diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 416f04a..dea1915 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", - "version": "0.7.1", - "description": "Initialize ITSM-bench workspaces, mine exact failure patterns, and record evidence-bound curations.", + "version": "0.8.0", + "description": "Prepare ITSM-bench workspaces, record hypotheses, and execute isolated candidates.", "author": { "name": "OpenFlyWheel" }, @@ -10,14 +10,14 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Mine and curate ITSM failure patterns", - "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, and persist evidence-bound cross-task curations without copying trace payloads.", + "shortDescription": "Execute isolated harness candidates", + "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, persist one evidence-backed hypothesis, execute one isolated candidate, and stop before admission without copying trace payloads.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], "defaultPrompt": [ "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.", - "When PROGRAM.md is ready, read it and continue the optimization loop until its declared goal or stop condition is met." + "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, execute one isolated candidate, and stop before admission." ] }, "mcpServers": "./.mcp.json" diff --git a/plugins/openflywheel/.mcp.json b/plugins/openflywheel/.mcp.json index 00d2fb5..b2cfe01 100644 --- a/plugins/openflywheel/.mcp.json +++ b/plugins/openflywheel/.mcp.json @@ -4,7 +4,7 @@ "command": "uvx", "args": [ "--from", - "git+https://github.com/divo12/OpenFlyWheel.git@b4c785d9472ca1d6c245c41fa4d9985f0cef97fc", + "git+https://github.com/divo12/OpenFlyWheel.git@9041db3c08a89df0fe9f8f2476a303b46dd2812a", "--with", "mcp>=1.13,<2", "openflywheel-mcp" diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index 288034d..c440f02 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -4,16 +4,19 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Improve the connected agent harness until the goal in `experiment_config.yaml` is met, -while respecting its quality, cost, latency, budget, and stopping constraints. +Record one evidence-backed hypothesis, execute its isolated candidate under the canonical +experiment policy, then stop before admission. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged -baseline. +baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an +existing terminal Harbor job was adopted). ## Authority -- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal, - benchmark, verifier, budget, and stopping conditions. +- The canonical experiment policy stored in the Git common control directory defines the + prepared branch and commits, exact editable surface, frozen controls, goal, benchmark, + verifier, budget, and stopping conditions. `experiment_config.yaml` is a human-readable copy, + not an internal authority interface. - The external verifier is authoritative for task outcome. The agent's completion claim is not proof of success. - Langfuse is the source of truth for trajectories, usage, cost, and latency. @@ -24,9 +27,9 @@ baseline. ## Editable and frozen surfaces -Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark, -held-out tasks, verifier, model, reasoning budget, observability identity, or this program -to improve a score. +Target only exact paths allowed by the canonical experiment policy. Edit them only in the +candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks, +verifier, model, reasoning budget, observability identity, or this program. Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control flow changes unless the evidence requires the combination. @@ -46,37 +49,22 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn expected improvement, and possible regressions. Stop if the evidence cannot distinguish between materially different changes. -### 4. Improve the harness +Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported +pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call +`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution. -Make the smallest change within the declared editable surface. Preserve frozen controls and -unrelated user changes. +### 4. Execute one candidate -### 5. Gate the change +Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling +candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated +worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the +returned candidate worktree, then call `execute_candidate` again with the identical request. -Run only the prepared experiment command and gates declared by the workspace. Compare -task-level verifier outcomes and report quality, cost, and latency separately. Missing or -errored trials remain visible and cannot disappear from the denominator. +Poll identical requests while the candidate is running. Retain its candidate ID, Git commit, +trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally, +change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree. -### 6. Keep or revert +## Package boundary -Keep the change only when the configured gate admits it. Otherwise revert only the current -iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken -the gate to admit a candidate. - -Commit each admitted improvement on the prepared `ofw/` branch before the -next iteration. Keep one hypothesis per commit and include `OFW-Experiment` and `OFW-Run` -trailers. Do not commit failed candidates, generated run artifacts, credentials, or changes -outside the editable surface. Do not push or open a pull request without explicit user -authorization. - -### 7. Repeat - -Return to step 2 with the newly recorded run. Stop when the configured goal is met, the -budget or iteration limit is exhausted, the no-improvement condition is reached, or required -authoritative evidence is unavailable. - -## Final report - -Report the accepted harness revision, verifier-backed quality, cost, latency, remaining -unverified trials, iteration count, and exact stopping reason. Do not claim improvement from -an agent-authored summary alone. +Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission: +do not gate, accept, merge, publish, push, or install the candidate. diff --git a/plugins/openflywheel/skills/hypothesis-former/SKILL.md b/plugins/openflywheel/skills/hypothesis-former/SKILL.md new file mode 100644 index 0000000..ee368cc --- /dev/null +++ b/plugins/openflywheel/skills/hypothesis-former/SKILL.md @@ -0,0 +1,25 @@ +--- +name: hypothesis-former +description: Record one OpenFlywheel harness hypothesis from one curated failure group and its exact supported pattern and diagnosis receipts. Use after failure curation and before any candidate edit; do not use with incomplete evidence or to modify harness files. +--- + +# Hypothesis Former + +Select one group from the retained failure-curation receipt. Use every diagnosis artifact ID in +that group, partitioned under its exact mined pattern IDs; do not add or omit IDs. Keep the global +diagnosis set at fifty IDs or fewer. Read the compact diagnoses, then propose one bounded +statement, rationale, expected effect, regression-risk list, component taxonomy, and one or more +exact target paths within the group's target component, explicit predicted task IDs, and at-risk +task IDs. Keep those prediction sets disjoint so the next iteration can falsify the decision. + +Call `record_hypothesis` once with the prepared worktree, experiment ID, current initialization +commit, curation ID, selected curation group ID, exact pattern-to-diagnosis assignments, explicit +predicted and at-risk task IDs, and proposed target. The tool reloads the curation, requires the complete selected group, recomputes +the patterns, and rejects missing, extra, misassigned, or inconclusive evidence. It also proves +each target is exactly present in the authoritative editable allowlist; `component_kind` must +match the curated component and never grants path access. + +Retain the returned hypothesis ID and artifact path. Stop before editing any harness file. On an +MCP timeout, retry the identical request once because publication is idempotent; if the retry +also times out, stop with unknown operation status. Do not query Langfuse, rewrite diagnoses, +broaden target paths, infer support, or begin candidate work while following this skill. diff --git a/plugins/openflywheel/skills/workspace-init/SKILL.md b/plugins/openflywheel/skills/workspace-init/SKILL.md index 54dae4b..3505c15 100644 --- a/plugins/openflywheel/skills/workspace-init/SKILL.md +++ b/plugins/openflywheel/skills/workspace-init/SKILL.md @@ -25,7 +25,10 @@ when the evidence supports one. Collect, in order: directories. 2. Optimization goal, primary metric, target, and stopping condition. Keep quality, cost, and latency constraints separate rather than hiding them in one average. -3. ITSM-bench root, Harbor executable, Harbor configuration, and expected task count. +3. ITSM-bench root, Harbor executable, Harbor configuration, and expected task count. If an + exact terminal Harbor job already exists at `benchmark_root/jobs/`, ask whether + to adopt it; set `reuse_existing_baseline=true` only after explicit confirmation so policy is + published without launching a duplicate baseline. 4. Experiment ID and maximum baseline duration. Read and report the frozen model from the Harbor configuration. `prepare_workspace` fixes diff --git a/pyproject.toml b/pyproject.toml index a3db39c..ceb748f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "openflywheel" -version = "0.7.1" +version = "0.8.0" description = "A governed self-improving agent harness" requires-python = ">=3.11" dependencies = [ diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 3c780bb..7287b86 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -13,6 +13,9 @@ from ofw.contracts import ComponentKind, Sha256Digest from ofw.evaluation import ( DeferredFailure, + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, EvidenceReference, FailureCuration, FailureCurationErrorCode, @@ -39,11 +42,33 @@ OutcomeScoreSubmission, OutcomeStoreObservation, OutcomeStoreStatus, + RunSide, TaskId, VerifierId, VerifierResult, VerifierVerdict, ) +from ofw.evolution import ( + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateFailure, + CandidateId, + CandidatePhase, + CandidateStatus, + FailurePatternReference, + FailurePatternReferenceInput, + HarnessChangeTarget, + HarnessChangeTargetInput, + HarnessHypothesis, + HypothesisErrorCode, + HypothesisFailure, + HypothesisId, + HypothesisObservation, + HypothesisStatus, + RecordHypothesisInput, +) from ofw.observability.langfuse import ( CollectionError, CollectionErrorCode, @@ -52,6 +77,9 @@ ) from ofw.observability.langfuse.domain import TraceId from ofw.preparation import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, PreparationErrorCode, PreparationPhase, PreparationStatus, @@ -60,11 +88,25 @@ ) __all__ = [ + "CandidateBlockerCode", + "CandidateErrorCode", + "CandidateExecutionInput", + "CandidateExecutionObservation", + "CandidateFailure", + "CandidateId", + "CandidatePhase", + "CandidateStatus", "CollectionError", "CollectionErrorCode", "ComponentKind", "DeferredFailure", "EvidenceReference", + "EvaluatedRunBlocker", + "EvaluatedRunReceipt", + "EvaluatedTaskReceipt", + "ExperimentPolicyErrorCode", + "ExperimentPolicyFailure", + "ExperimentPolicySnapshot", "FailureCuration", "FailureCurationErrorCode", "FailureCurationFailure", @@ -80,8 +122,18 @@ "FailurePatternMiningStatus", "FailurePatternOrdering", "FailurePatternSummary", + "FailurePatternReference", + "FailurePatternReferenceInput", "FailureSource", "FailureType", + "HarnessChangeTarget", + "HarnessChangeTargetInput", + "HarnessHypothesis", + "HypothesisErrorCode", + "HypothesisFailure", + "HypothesisId", + "HypothesisObservation", + "HypothesisStatus", "Langfuse", "LangfuseOtelSpanAttributes", "LangfuseOutcomeStore", @@ -94,10 +146,12 @@ "OutcomeScoreSubmission", "OutcomeStoreObservation", "OutcomeStoreStatus", + "RunSide", "PreparationErrorCode", "PreparationPhase", "PreparationStatus", "PrepareWorkspaceInput", + "RecordHypothesisInput", "Sha256Digest", "TaskId", "TraceId", diff --git a/src/ofw/evaluation/__init__.py b/src/ofw/evaluation/__init__.py index d39931e..980e5f9 100644 --- a/src/ofw/evaluation/__init__.py +++ b/src/ofw/evaluation/__init__.py @@ -32,10 +32,14 @@ OutcomeStoreStatus, ) from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, EvidenceReference, OutcomeErrorCode, OutcomeEvaluation, OutcomeEvaluationError, + RunSide, TaskId, VerifierId, VerifierResult, @@ -45,6 +49,9 @@ __all__ = [ "DeferredFailure", "EvidenceReference", + "EvaluatedRunBlocker", + "EvaluatedRunReceipt", + "EvaluatedTaskReceipt", "FailureCuration", "FailureCurationErrorCode", "FailureCurationFailure", @@ -67,6 +74,7 @@ "OutcomeErrorCode", "OutcomeEvaluation", "OutcomeEvaluationError", + "RunSide", "OutcomeScoreSubmission", "OutcomeStoreObservation", "OutcomeStoreStatus", diff --git a/src/ofw/evaluation/failure_curation.py b/src/ofw/evaluation/failure_curation.py index f5e2b2b..6ae94ec 100644 --- a/src/ofw/evaluation/failure_curation.py +++ b/src/ofw/evaluation/failure_curation.py @@ -16,11 +16,11 @@ from ofw.observability.langfuse.domain import ObservationId, ScoreId, TraceId _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -_PATTERN_KEY_PATTERN = r"[a-z0-9]+(?:-[a-z0-9]+)*" -_TEXT_PATTERN = r"[^\x00]+" +_PATTERN_KEY_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" +_TEXT_PATTERN = r"^[^\x00]+$" ArtifactIdentifier = Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)] -DigestValue = Annotated[str, Field(pattern=r"sha256:[0-9a-f]{64}")] +DigestValue = Annotated[str, Field(pattern=r"^sha256:[0-9a-f]{64}$")] PatternKey = Annotated[ str, Field(min_length=1, max_length=80, pattern=_PATTERN_KEY_PATTERN), @@ -292,6 +292,17 @@ class FailureCurationArtifact(StrictModel): groups: tuple[FailureGroupArtifact, ...] = Field(max_length=25) deferred: tuple[DeferredFailureArtifact, ...] = Field(max_length=50) + def require_valid_identity(self) -> None: + _require_group_artifact_identities(self.groups) + _require_complete_partition(self.source_artifact_ids, _artifact_assignment_ids(self)) + expected = _failure_curation_id( + self.source_artifact_ids, + tuple(group.group_id for group in self.groups), + _artifact_deferred_evidence(self.deferred), + ) + if self.curation_id != expected: + raise ValueError("curation_id does not match canonical curation content") + @classmethod def from_curation(cls, curation: FailureCuration) -> FailureCurationArtifact: return cls( @@ -490,36 +501,108 @@ def _group_id( item: FailureGroupInput, issue_type: FailureType, members: tuple[FailureGroupMember, ...], +) -> str: + evidence = tuple( + (member.artifact_id, member.artifact_digest.value) for member in members + ) + return _failure_group_id( + item.pattern_key, + item.title, + item.mechanism, + item.prevention, + item.target_component, + issue_type, + evidence, + ) + + +def _curation_id( + source_ids: tuple[str, ...], + groups: tuple[FailureGroup, ...], + deferred: tuple[DeferredFailure, ...], +) -> str: + deferred_evidence = tuple( + (item.source.artifact_id, item.source.artifact_digest.value, item.reason) + for item in deferred + ) + return _failure_curation_id( + source_ids, + tuple(group.id for group in groups), + deferred_evidence, + ) + + +def _failure_group_id( + pattern_key: str, + title: str, + mechanism: str, + prevention: str, + target_component: ComponentKind, + issue_type: FailureType, + members: tuple[tuple[str, str], ...], ) -> str: identity = "\0".join( ( "ofw.failure-group", - item.pattern_key, - item.title, - item.mechanism, - item.prevention, - item.target_component.value, + pattern_key, + title, + mechanism, + prevention, + target_component.value, issue_type.value, - *(f"{member.artifact_id}:{member.artifact_digest.value}" for member in members), + *(f"{artifact_id}:{digest}" for artifact_id, digest in members), ) ) return str(uuid5(NAMESPACE_URL, identity)) -def _curation_id( +def _require_group_artifact_identity(group: FailureGroupArtifact) -> None: + members = tuple((item.artifact_id, item.artifact_digest) for item in group.members) + expected = _failure_group_id( + group.pattern_key, + group.title, + group.mechanism, + group.prevention, + group.target_component, + group.issue_type, + members, + ) + if group.group_id != expected: + raise ValueError("group_id does not match canonical group content") + + +def _require_group_artifact_identities(groups: tuple[FailureGroupArtifact, ...]) -> None: + for group in groups: + _require_group_artifact_identity(group) + + +def _artifact_assignment_ids(artifact: FailureCurationArtifact) -> tuple[str, ...]: + grouped = tuple( + item.artifact_id for group in artifact.groups for item in group.members + ) + return grouped + tuple(item.source.artifact_id for item in artifact.deferred) + + +def _artifact_deferred_evidence( + deferred: tuple[DeferredFailureArtifact, ...], +) -> tuple[tuple[str, str, str], ...]: + return tuple( + (item.source.artifact_id, item.source.artifact_digest, item.reason) + for item in deferred + ) + + +def _failure_curation_id( source_ids: tuple[str, ...], - groups: tuple[FailureGroup, ...], - deferred: tuple[DeferredFailure, ...], + group_ids: tuple[str, ...], + deferred: tuple[tuple[str, str, str], ...], ) -> str: identity = "\0".join( ( "ofw.failure-curation", *source_ids, - *(group.id for group in groups), - *( - f"{item.source.artifact_id}:{item.source.artifact_digest.value}:{item.reason}" - for item in deferred - ), + *group_ids, + *(f"{artifact_id}:{digest}:{reason}" for artifact_id, digest, reason in deferred), ) ) return str(uuid5(NAMESPACE_URL, identity)) diff --git a/src/ofw/evaluation/failure_patterns.py b/src/ofw/evaluation/failure_patterns.py index 10a4f19..f4dd7e5 100644 --- a/src/ofw/evaluation/failure_patterns.py +++ b/src/ofw/evaluation/failure_patterns.py @@ -15,7 +15,7 @@ from ofw.evaluation.failure import FailureDiagnosis, FailureEvidenceStatus, FailureType _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -_PATTERN_ID_PATTERN = r"sha256:[0-9a-f]{64}" +_PATTERN_ID_PATTERN = r"^sha256:[0-9a-f]{64}$" _ABSOLUTE_PATH = re.compile(r"(?:/[\w.+-]+){2,}") _OPAQUE_ID = re.compile( r"\b(?:(?:request|trace|span|observation|score|session|run)(?:[_-]?id)?|rq)" diff --git a/src/ofw/evaluation/failure_workspace.py b/src/ofw/evaluation/failure_workspace.py index 6669a45..c65d47e 100644 --- a/src/ofw/evaluation/failure_workspace.py +++ b/src/ofw/evaluation/failure_workspace.py @@ -3,17 +3,13 @@ from __future__ import annotations import hashlib -import os import re -import stat -from collections.abc import Iterator -from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta from enum import StrEnum from pathlib import Path from typing import Annotated, Literal, Never, Protocol -from uuid import NAMESPACE_URL, uuid4, uuid5 +from uuid import NAMESPACE_URL, uuid5 from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator @@ -46,8 +42,16 @@ VerifierVerdict, ) from ofw.observability.langfuse.domain import ObservationId, ScoreId, TraceId +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_child_directory, + open_directory_chain, + publish_idempotent, + read_bounded, +) -_IDENTIFIER_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" +_IDENTIFIER_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$" _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" _ARTIFACT_ID = re.compile(_ARTIFACT_ID_PATTERN) _ARTIFACT_LIMIT_BYTES = 64 * 1024 @@ -56,9 +60,6 @@ _CURATION_DIRECTORY = "failure-curations" _IGNORE_CONTENT = "*\n" _WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") -_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW -_CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW -_READ_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK Identifier = Annotated[ str, @@ -173,8 +174,9 @@ class FailureRecordObservation(StrictModel): class FailureArtifact(StrictModel): - schema_version: Literal[1] = 1 + schema_version: Literal[2] = 2 artifact_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + content_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") trace_id: Identifier task_id: Identifier verifier_id: Identifier @@ -195,9 +197,11 @@ class FailureArtifact(StrictModel): @model_validator(mode="after") def validate_domain_contract(self) -> FailureArtifact: try: - self.to_diagnosis() + diagnosis = self.to_diagnosis() except (FailureDiagnosisError, OutcomeEvaluationError) as error: raise ValueError("invalid failure artifact") from error + if self.content_digest != _diagnosis_content_digest(diagnosis): + raise ValueError("failure artifact content digest mismatch") return self @classmethod @@ -209,6 +213,7 @@ def from_diagnosis( outcome = diagnosis.outcome return cls( artifact_id=artifact_id, + content_digest=_diagnosis_content_digest(diagnosis), trace_id=outcome.trace_id.value, task_id=outcome.task_id.value, verifier_id=outcome.verifier_id.value, @@ -262,13 +267,6 @@ class FailureArtifactReceipt: relative_path: Path -@dataclass(frozen=True, slots=True) -class _DirectoryChainIdentity: - root: tuple[int, int] - workspace: tuple[int, int] - artifacts: tuple[int, int] - - class FailureWorkspaceErrorCode(StrEnum): INVALID_WORKSPACE = "invalid_workspace" ARTIFACT_CONFLICT = "artifact_conflict" @@ -338,10 +336,18 @@ def read( ) -> tuple[FailureDiagnosisRecord, ...]: try: return self._read(root, artifact_ids) - except FailurePatternMiningError: - raise + except FileNotFoundError: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND, + _FAILURE_DIRECTORY, + ) from None except FailureWorkspaceFailure as error: raise _pattern_read_error(error) from None + except SafeFileFailure: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_WORKSPACE, + _WORKSPACE_DIRECTORY, + ) from None except (OSError, RuntimeError, UnicodeError): raise FailurePatternMiningError( FailurePatternMiningErrorCode.READ_FAILED, @@ -354,16 +360,10 @@ def _read( artifact_ids: tuple[str, ...], ) -> tuple[FailureDiagnosisRecord, ...]: prepared_root = _prepared_root(root) - workspace, failures = _workspace_artifact_paths(prepared_root, _FAILURE_DIRECTORY) - directory_identity = _existing_workspace_directories( + with open_directory_chain( prepared_root, - workspace, - failures, - ) - with _artifact_directory_handle( - prepared_root, - _FAILURE_DIRECTORY, - directory_identity, + (_WORKSPACE_DIRECTORY, _FAILURE_DIRECTORY), + create=False, ) as directory: return tuple(_read_artifact(directory, artifact_id) for artifact_id in artifact_ids) @@ -374,8 +374,6 @@ class FileFailureCurationWorkspace: def load(self, root: Path, artifact_ids: tuple[str, ...]) -> tuple[FailureSource, ...]: try: return self._load(root, artifact_ids) - except FailureCurationFailure: - raise except FailureWorkspaceFailure as error: code = ( FailureCurationErrorCode.SOURCE_INVALID @@ -383,6 +381,11 @@ def load(self, root: Path, artifact_ids: tuple[str, ...]) -> tuple[FailureSource else FailureCurationErrorCode.INVALID_WORKSPACE ) raise FailureCurationFailure(code, error.subject) from None + except SafeFileFailure: + raise FailureCurationFailure( + FailureCurationErrorCode.INVALID_WORKSPACE, + _WORKSPACE_DIRECTORY, + ) from None except (OSError, RuntimeError, UnicodeError): raise FailureCurationFailure( FailureCurationErrorCode.SOURCE_INVALID, @@ -414,29 +417,19 @@ def _load( artifact_ids: tuple[str, ...], ) -> tuple[FailureSource, ...]: prepared_root = _prepared_root(root) - workspace, failures = _workspace_artifact_paths(prepared_root, _FAILURE_DIRECTORY) try: - identity = _existing_directory_identity(prepared_root, workspace, failures) - with _artifact_directory_handle( + with open_directory_chain( prepared_root, - _FAILURE_DIRECTORY, - identity, + (_WORKSPACE_DIRECTORY, _FAILURE_DIRECTORY), + create=False, ) as directory: return tuple( _read_failure_source(directory, artifact_id) for artifact_id in artifact_ids ) except FileNotFoundError: - missing = next( - ( - artifact_id - for artifact_id in artifact_ids - if not (failures / f"{artifact_id}.json").exists() - ), - artifact_ids[0], - ) raise FailureCurationFailure( FailureCurationErrorCode.SOURCE_NOT_FOUND, - missing, + artifact_ids[0], ) from None @@ -508,6 +501,50 @@ def _artifact_id(diagnosis: FailureDiagnosis) -> str: return str(uuid5(NAMESPACE_URL, identity)) +def _diagnosis_content_digest(diagnosis: FailureDiagnosis) -> str: + outcome = diagnosis.outcome + score = _required_score(outcome) + values = ( + "ofw.failure-artifact/1", + outcome.trace_id.value, + outcome.task_id.value, + outcome.verifier_id.value, + outcome.evaluated_at.isoformat(), + score.hex(), + str(len(outcome.evidence)), + *(reference.value for reference in outcome.evidence), + diagnosis.outcome_score_id.value, + diagnosis.evidence_status.value, + _optional_enum(diagnosis.issue_type), + diagnosis.expected_outcome, + diagnosis.actual_outcome, + _optional_identifier(diagnosis.critical_observation_id), + str(len(diagnosis.evidence_observation_ids)), + *(identifier.value for identifier in diagnosis.evidence_observation_ids), + _optional_text(diagnosis.root_cause), + _optional_text(diagnosis.counterfactual_action), + _optional_text(diagnosis.inconclusive_reason), + ) + digest = hashlib.sha256() + for value in values: + encoded = value.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def _optional_enum(value: FailureType | None) -> str: + return "0" if value is None else f"1{value.value}" + + +def _optional_identifier(value: ObservationId | None) -> str: + return "0" if value is None else f"1{value.value}" + + +def _optional_text(value: str | None) -> str: + return "0" if value is None else f"1{value}" + + def _prepared_root(root: Path) -> Path: resolved = _resolve_root(root) if not _is_prepared_root(resolved): @@ -526,36 +563,33 @@ def _is_prepared_root(root: Path) -> bool: return all((root / name).is_file() for name in _WORKSPACE_MARKERS) -def _workspace_artifact_paths(root: Path, directory_name: str) -> tuple[Path, Path]: - workspace = root / _WORKSPACE_DIRECTORY - failures = workspace / directory_name - _require_contained(root, workspace.resolve(strict=False)) - _require_contained(root, failures.resolve(strict=False)) - _require_directory_if_present(workspace) - return workspace, failures - - def _store_artifact(root: Path, directory_name: str, artifact_id: str, content: bytes) -> Path: prepared_root = _prepared_root(root) - workspace, artifacts = _workspace_artifact_paths(prepared_root, directory_name) _validate_artifact_size(content) - identity = _prepare_workspace_directories(prepared_root, workspace, artifacts) - path = artifacts / f"{artifact_id}.json" - with _artifact_directory_handle(prepared_root, directory_name, identity) as directory: - _publish_or_validate(directory, path.name, content, artifact_id) - return path.relative_to(prepared_root) - - -def _require_directory_if_present(path: Path) -> None: - if path.exists() and not path.is_dir(): - _invalid_workspace(_WORKSPACE_DIRECTORY) - - -def _require_contained(root: Path, path: Path) -> None: + name = f"{artifact_id}.json" try: - path.relative_to(root) - except ValueError: - _invalid_workspace(_WORKSPACE_DIRECTORY) + with open_directory_chain( + prepared_root, + (_WORKSPACE_DIRECTORY,), + create=True, + ) as workspace: + publish_idempotent( + workspace, + ".gitignore", + _IGNORE_CONTENT.encode("utf-8"), + maximum_bytes=16, + subject=_WORKSPACE_DIRECTORY, + ) + with open_child_directory(workspace, directory_name, create=True) as directory: + _publish_or_validate(directory, name, content, artifact_id) + except SafeFileFailure as error: + code = ( + FailureWorkspaceErrorCode.INVALID_WORKSPACE + if error.code is SafeFileErrorCode.INVALID_FILE + else FailureWorkspaceErrorCode.WRITE_FAILED + ) + raise FailureWorkspaceFailure(code, artifact_id) from None + return Path(_WORKSPACE_DIRECTORY, directory_name, name) def _invalid_workspace(subject: str) -> Never: @@ -573,58 +607,6 @@ def _validate_artifact_size(content: bytes) -> None: ) -def _prepare_workspace_directories( - root: Path, - workspace: Path, - failures: Path, -) -> _DirectoryChainIdentity: - failures.mkdir(parents=True, exist_ok=True) - _require_contained(root, workspace.resolve(strict=True)) - _require_contained(root, failures.resolve(strict=True)) - with _directory_handle(workspace) as directory: - _require_directory_identity(directory, workspace) - _write_ignore_file(directory) - return _DirectoryChainIdentity( - root=_path_identity(root), - workspace=_path_identity(workspace), - artifacts=_path_identity(failures), - ) - - -def _existing_directory_identity( - root: Path, - workspace: Path, - artifacts: Path, -) -> _DirectoryChainIdentity: - _require_contained(root, workspace.resolve(strict=True)) - _require_contained(root, artifacts.resolve(strict=True)) - return _DirectoryChainIdentity( - root=_path_identity(root), - workspace=_path_identity(workspace), - artifacts=_path_identity(artifacts), - ) - - -def _existing_workspace_directories( - root: Path, - workspace: Path, - failures: Path, -) -> _DirectoryChainIdentity: - if not workspace.is_dir() or not failures.is_dir(): - raise FailurePatternMiningError( - FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND, - _FAILURE_DIRECTORY, - ) - return _existing_directory_identity(root, workspace, failures) - - -def _write_ignore_file(directory: int) -> None: - try: - _write_new_file(directory, ".gitignore", _IGNORE_CONTENT.encode("utf-8")) - except FileExistsError: - return - - def _publish_or_validate( directory: int, name: str, @@ -632,57 +614,25 @@ def _publish_or_validate( artifact_id: str, ) -> None: try: - _publish_new_file(directory, name, expected) - except FileExistsError: - _validate_existing(directory, name, expected, artifact_id) - - -def _publish_new_file(directory: int, name: str, content: bytes) -> None: - temporary_name = f".ofw-{uuid4().hex}.tmp" - published = False - try: - _write_new_file(directory, temporary_name, content) - os.link( - temporary_name, + publish_idempotent( + directory, name, - src_dir_fd=directory, - dst_dir_fd=directory, - follow_symlinks=False, - ) - published = True - finally: - _unlink_if_present(directory, temporary_name) - if published: - os.fsync(directory) - - -def _write_new_file(directory: int, name: str, content: bytes) -> None: - descriptor = os.open(name, _CREATE_FILE_FLAGS, 0o600, dir_fd=directory) - with os.fdopen(descriptor, "wb") as stream: - stream.write(content) - stream.flush() - os.fsync(stream.fileno()) - - -def _unlink_if_present(directory: int, name: str) -> None: - try: - os.unlink(name, dir_fd=directory) - except FileNotFoundError: - return - - -def _validate_existing( - directory: int, - name: str, - expected: bytes, - artifact_id: str, -) -> None: - actual = _read_existing(directory, name, artifact_id) - if actual != expected: - raise FailureWorkspaceFailure( - FailureWorkspaceErrorCode.ARTIFACT_CONFLICT, - artifact_id, + expected, + maximum_bytes=_ARTIFACT_LIMIT_BYTES, + subject=artifact_id, ) + except SafeFileFailure as error: + if error.code is SafeFileErrorCode.CONFLICT: + raise FailureWorkspaceFailure( + FailureWorkspaceErrorCode.ARTIFACT_CONFLICT, + artifact_id, + ) from None + if error.code is SafeFileErrorCode.TOO_LARGE: + raise FailureWorkspaceFailure( + FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE, + artifact_id, + ) from None + raise OSError("unsafe failure artifact") from None def _read_artifact(directory: int, artifact_id: str) -> FailureDiagnosisRecord: @@ -735,114 +685,17 @@ def _pattern_read_error(error: FailureWorkspaceFailure) -> FailurePatternMiningE def _read_existing(directory: int, name: str, artifact_id: str) -> bytes: - descriptor = os.open(name, _READ_FILE_FLAGS, dir_fd=directory) try: - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise OSError("failure artifact is not a regular file") - except OSError: - os.close(descriptor) - raise - with os.fdopen(descriptor, "rb") as stream: - content = stream.read(_ARTIFACT_LIMIT_BYTES + 1) - if len(content) > _ARTIFACT_LIMIT_BYTES: - raise FailureWorkspaceFailure( - FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE, - artifact_id, - ) - return content - - -@contextmanager -def _directory_handle(path: Path) -> Iterator[int]: - descriptor = os.open(path, _DIRECTORY_FLAGS) - try: - yield descriptor - finally: - os.close(descriptor) - - -@contextmanager -def _child_directory_handle(parent: int, name: str) -> Iterator[int]: - descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent) - try: - yield descriptor - finally: - os.close(descriptor) - - -@contextmanager -def _artifact_directory_handle( - root: Path, - directory_name: str, - expected: _DirectoryChainIdentity, -) -> Iterator[int]: - with ( - _directory_handle(root) as root_directory, - _child_directory_handle(root_directory, _WORKSPACE_DIRECTORY) as workspace, - _child_directory_handle(workspace, directory_name) as artifacts, - ): - _require_directory_chain( - root, - root_directory, - workspace, - directory_name, - artifacts, - expected, - ) - yield artifacts - _require_directory_chain( - root, - root_directory, - workspace, - directory_name, - artifacts, - expected, + return read_bounded( + directory, + name, + maximum_bytes=_ARTIFACT_LIMIT_BYTES, + subject=artifact_id, ) - - -def _require_directory_chain( - root: Path, - root_directory: int, - workspace: int, - directory_name: str, - artifacts: int, - expected: _DirectoryChainIdentity, -) -> None: - _require_directory_identity(root_directory, root, expected.root) - _require_child_identity(root_directory, _WORKSPACE_DIRECTORY, workspace, expected.workspace) - _require_child_identity(workspace, directory_name, artifacts, expected.artifacts) - - -def _require_directory_identity( - descriptor: int, - path: Path, - expected: tuple[int, int] | None = None, -) -> None: - opened = _descriptor_identity(descriptor) - current = _path_identity(path) - if opened != current or (expected is not None and opened != expected): - raise OSError("workspace directory changed during failure recording") - - -def _require_child_identity( - parent: int, - name: str, - descriptor: int, - expected: tuple[int, int], -) -> None: - opened = _descriptor_identity(descriptor) - current = _stat_identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) - if opened != current or opened != expected: - raise OSError("workspace directory changed during failure recording") - - -def _descriptor_identity(descriptor: int) -> tuple[int, int]: - return _stat_identity(os.fstat(descriptor)) - - -def _path_identity(path: Path) -> tuple[int, int]: - return _stat_identity(os.stat(path, follow_symlinks=False)) - - -def _stat_identity(value: os.stat_result) -> tuple[int, int]: - return value.st_dev, value.st_ino + except SafeFileFailure as error: + if error.code is SafeFileErrorCode.TOO_LARGE: + raise FailureWorkspaceFailure( + FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE, + artifact_id, + ) from None + raise OSError("unsafe failure artifact") from None diff --git a/src/ofw/evaluation/outcome.py b/src/ofw/evaluation/outcome.py index b7f53c2..b938181 100644 --- a/src/ofw/evaluation/outcome.py +++ b/src/ofw/evaluation/outcome.py @@ -2,18 +2,27 @@ from __future__ import annotations +import hashlib import math import re from dataclasses import dataclass from datetime import datetime, timedelta from enum import StrEnum +from pydantic import BaseModel, ConfigDict, Field, StrictStr, model_validator + from ofw.observability.langfuse.domain import TraceId _IDENTIFIER_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]*") _IDENTIFIER_LIMIT = 256 _EVIDENCE_LIMIT = 10 _EVIDENCE_VALUE_LIMIT = 1024 +_RUN_ID_LIMIT = 256 +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +_RUN_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$" +_RUN_METRIC_LIMIT = 172800.0 +_COST_LIMIT = 1_000_000.0 class OutcomeErrorCode(StrEnum): @@ -43,6 +52,168 @@ class VerifierVerdict(StrEnum): ERROR = "error" +class RunSide(StrEnum): + ACCEPTED = "accepted" + CANDIDATE = "candidate" + + +class _ReceiptModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EvaluatedTaskReceipt(_ReceiptModel): + task_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + trace_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + score_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + verdict: VerifierVerdict + verifier_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + normalized_score: float | None = None + cost_usd: float | None = None + latency_seconds: float | None = None + + @model_validator(mode="after") + def validate_metrics(self) -> EvaluatedTaskReceipt: + _validate_run_metric(self.normalized_score, 0.0, 1.0, "normalized_score") + _validate_run_metric(self.cost_usd, 0.0, _COST_LIMIT, "cost_usd") + _validate_run_metric( + self.latency_seconds, + 0.0, + _RUN_METRIC_LIMIT, + "latency_seconds", + ) + _validate_verdict_score(self.verdict, self.normalized_score) + return self + + +class EvaluatedRunBlocker(_ReceiptModel): + task_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + code: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + subject: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + + +class EvaluatedRunReceipt(_ReceiptModel): + receipt_id: StrictStr = Field(pattern=_DIGEST_PATTERN) + run_id: StrictStr = Field( + min_length=1, + max_length=_RUN_ID_LIMIT, + pattern=_RUN_ID_PATTERN, + ) + side: RunSide + policy_digest: StrictStr = Field(pattern=_DIGEST_PATTERN) + controls_digest: StrictStr = Field(pattern=_DIGEST_PATTERN) + evaluated_commit: StrictStr = Field(pattern=_COMMIT_PATTERN) + evaluated_tree: StrictStr = Field(pattern=_COMMIT_PATTERN) + task_ids: tuple[StrictStr, ...] = Field(min_length=1, max_length=500) + outcome_receipts: tuple[EvaluatedTaskReceipt, ...] = Field(max_length=500) + blockers: tuple[EvaluatedRunBlocker, ...] = Field(max_length=500) + + @classmethod + def build( + cls, + *, + run_id: str, + side: RunSide, + policy_digest: str, + controls_digest: str, + evaluated_commit: str, + evaluated_tree: str, + task_ids: tuple[str, ...], + outcome_receipts: tuple[EvaluatedTaskReceipt, ...], + blockers: tuple[EvaluatedRunBlocker, ...], + ) -> EvaluatedRunReceipt: + draft = cls.model_construct( + receipt_id="sha256:" + "0" * 64, + run_id=run_id, + side=side, + policy_digest=policy_digest, + controls_digest=controls_digest, + evaluated_commit=evaluated_commit, + evaluated_tree=evaluated_tree, + task_ids=task_ids, + outcome_receipts=outcome_receipts, + blockers=blockers, + ) + return cls( + receipt_id=draft.recomputed_id(), + run_id=run_id, + side=side, + policy_digest=policy_digest, + controls_digest=controls_digest, + evaluated_commit=evaluated_commit, + evaluated_tree=evaluated_tree, + task_ids=task_ids, + outcome_receipts=outcome_receipts, + blockers=blockers, + ) + + def recomputed_id(self) -> str: + canonical = self.model_dump_json(exclude={"receipt_id"}) + return f"sha256:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()}" + + @model_validator(mode="after") + def validate_identity_and_partition(self) -> EvaluatedRunReceipt: + if self.receipt_id != self.recomputed_id(): + raise ValueError("receipt_id does not match canonical receipt content") + task_ids = self.task_ids + _validate_unique_ids(task_ids, "task_ids") + outcome_ids = tuple(item.task_id for item in self.outcome_receipts) + blocker_ids = tuple(item.task_id for item in self.blockers) + all_result_ids = outcome_ids + blocker_ids + _validate_partition(all_result_ids, task_ids) + _validate_result_order(outcome_ids, task_ids) + _validate_result_order(blocker_ids, task_ids) + return self + + +def _validate_verdict_score(verdict: VerifierVerdict, score: float | None) -> None: + expected = ( + 1.0 + if verdict is VerifierVerdict.PASS + else 0.0 + if verdict is VerifierVerdict.FAIL + else None + ) + if score != expected: + raise ValueError("normalized_score does not match verdict") + + +def _validate_partition( + result_ids: tuple[str, ...], + task_ids: tuple[str, ...], +) -> None: + _validate_unique_ids(result_ids, "outcomes and blockers") + if set(result_ids) != set(task_ids): + raise ValueError("outcomes and blockers must partition task_ids") + + +def _validate_unique_ids(values: tuple[str, ...], field: str) -> None: + if len(set(values)) != len(values): + raise ValueError(f"{field} must be unique") + + +def _validate_result_order(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> None: + if not _follows_task_order(result_ids, task_ids): + raise ValueError("outcomes and blockers must follow task_ids order") + + +def _validate_run_metric( + value: float | None, + minimum: float, + maximum: float, + field: str, +) -> None: + if value is not None and (not math.isfinite(value) or not minimum <= value <= maximum): + raise ValueError(f"{field} is outside its finite bounds") + + +def _follows_task_order( + result_ids: tuple[str, ...], + task_ids: tuple[str, ...], +) -> bool: + positions = tuple(task_ids.index(task_id) for task_id in result_ids) + return positions == tuple(sorted(positions)) + + @dataclass(frozen=True, slots=True) class EvidenceReference: value: str diff --git a/src/ofw/evolution/__init__.py b/src/ofw/evolution/__init__.py new file mode 100644 index 0000000..2d6146a --- /dev/null +++ b/src/ofw/evolution/__init__.py @@ -0,0 +1,57 @@ +"""Prepared-experiment evolution contracts.""" + +from ofw.evolution.candidate import ( + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateFailure, + CandidateId, + CandidatePhase, + CandidateStatus, +) +from ofw.evolution.candidate_git import CandidateGitGateway +from ofw.evolution.candidate_langfuse import LangfuseCandidateTraceLocator +from ofw.evolution.candidate_service import CandidateExecutionService +from ofw.evolution.hypothesis import ( + FailurePatternReference, + FailurePatternReferenceInput, + HarnessChangeTarget, + HarnessChangeTargetInput, + HarnessHypothesis, + HypothesisErrorCode, + HypothesisFailure, + HypothesisId, + HypothesisObservation, + HypothesisService, + HypothesisStatus, + RecordHypothesisInput, +) +from ofw.evolution.hypothesis_repository import FileHypothesisRepository + +__all__ = [ + "CandidateBlockerCode", + "CandidateErrorCode", + "CandidateExecutionInput", + "CandidateExecutionObservation", + "CandidateExecutionService", + "CandidateFailure", + "CandidateGitGateway", + "CandidateId", + "CandidatePhase", + "CandidateStatus", + "FailurePatternReference", + "FailurePatternReferenceInput", + "FileHypothesisRepository", + "HarnessChangeTarget", + "HarnessChangeTargetInput", + "HarnessHypothesis", + "HypothesisErrorCode", + "HypothesisFailure", + "HypothesisId", + "HypothesisObservation", + "HypothesisService", + "HypothesisStatus", + "LangfuseCandidateTraceLocator", + "RecordHypothesisInput", +] diff --git a/src/ofw/evolution/candidate.py b/src/ofw/evolution/candidate.py new file mode 100644 index 0000000..8f8232d --- /dev/null +++ b/src/ofw/evolution/candidate.py @@ -0,0 +1,256 @@ +"""Immutable candidate-execution contracts.""" + +from __future__ import annotations + +import hashlib +import math +import re +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from pathlib import Path +from typing import Literal, Protocol + +from pydantic import Field, field_validator + +from ofw.evaluation.langfuse import OutcomeScoreSubmission +from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + OutcomeEvaluation, +) +from ofw.evolution.hypothesis import StrictModel +from ofw.preparation.contracts import ( + ExperimentControls, + ExperimentRun, + ExperimentSummary, + PathValue, + contained_relative_path, +) +from ofw.preparation.policy import ExperimentPolicySnapshot + +_DIGEST_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") +_IDENTIFIER_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]*") + + +class _CandidateIdentity(StrictModel): + schema_version: Literal[1] = 1 + policy_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + hypothesis_id: str = Field(pattern=r"sha256:[0-9a-f]{64}") + source_commit: str = Field(pattern=r"[0-9a-f]{40}") + candidate_tree: str = Field(pattern=r"[0-9a-f]{40}") + controls_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + + +@dataclass(frozen=True, slots=True) +class CandidateId: + value: str + + def __post_init__(self) -> None: + if _DIGEST_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid candidate id") + + def __str__(self) -> str: + return self.value + + @classmethod + def build( + cls, + *, + policy_digest: str, + hypothesis_id: str, + source_commit: str, + candidate_tree: str, + controls_digest: str, + ) -> CandidateId: + identity = _CandidateIdentity( + policy_digest=policy_digest, + hypothesis_id=hypothesis_id, + source_commit=source_commit, + candidate_tree=candidate_tree, + controls_digest=controls_digest, + ) + digest = hashlib.sha256(identity.model_dump_json().encode("utf-8")).hexdigest() + return cls(f"sha256:{digest}") + + +def candidate_policy_digest(policy: ExperimentPolicySnapshot) -> str: + digest = hashlib.sha256(policy.model_dump_json().encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +class CandidateErrorCode(StrEnum): + INVALID_WORKSPACE = "invalid_workspace" + WORKTREE_EXISTS = "worktree_exists" + STALE_COMMIT = "stale_commit" + STALE_POLICY = "stale_policy" + EMPTY_CANDIDATE = "empty_candidate" + OUT_OF_SCOPE = "out_of_scope" + UNSAFE_PATH = "unsafe_path" + MANAGED_PATH = "managed_path" + CREDENTIAL_PATH = "credential_path" + CONTROLS_DRIFT = "controls_drift" + REQUEST_CONFLICT = "request_conflict" + MISSING_ENVIRONMENT = "missing_environment" + LAUNCH_FAILED = "launch_failed" + CANDIDATE_TIMEOUT = "candidate_timeout" + INVALID_RESULT = "invalid_result" + OUTCOME_STORE_FAILED = "outcome_store_failed" + GIT_FAILED = "git_failed" + + +class CandidateFailure(Exception): + __slots__ = ("code", "subject") + + def __init__(self, code: CandidateErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +@dataclass(frozen=True, slots=True) +class CandidateWorkspace: + accepted_root: Path + worktree_path: Path + source_commit: str + + +@dataclass(frozen=True, slots=True) +class CandidateTree: + tree_id: str + changed_paths: tuple[Path, ...] + + +@dataclass(frozen=True, slots=True) +class CandidateCommit: + commit: str + + +class CandidatePhase(StrEnum): + EDITING = "editing" + RUNNING = "running" + COMPLETE = "complete" + FAILED = "failed" + + +class CandidateStatus(StrEnum): + SUCCESS = "success" + WARNING = "warning" + ERROR = "error" + + +class CandidateBlockerCode(StrEnum): + TRACE_NOT_FOUND = "trace_not_found" + TRACE_AMBIGUOUS = "trace_ambiguous" + UNVERIFIED = "unverified" + UNSUPPORTED_REWARD = "unsupported_reward" + + +class CandidateExecutionInput(StrictModel): + workspace_root: PathValue + worktree_parent: PathValue + benchmark_root: PathValue + harbor_executable: PathValue + harbor_config: PathValue + experiment_id: str = Field(pattern=r"[a-z0-9]+(?:-[a-z0-9]+)*", max_length=80) + hypothesis_id: str = Field(pattern=r"sha256:[0-9a-f]{64}") + + @field_validator( + "workspace_root", + "worktree_parent", + "benchmark_root", + "harbor_executable", + ) + @classmethod + def validate_absolute(cls, value: Path) -> Path: + if not value.is_absolute(): + raise ValueError("path must be absolute") + return value + + @field_validator("harbor_config") + @classmethod + def validate_config(cls, value: Path) -> Path: + return contained_relative_path(value, "harbor_config") + + +class CandidateExecutionObservation(StrictModel): + status: CandidateStatus + summary: str = Field(min_length=1, max_length=256) + next_actions: tuple[str, ...] = Field(max_length=2) + artifacts: tuple[str, ...] = Field(max_length=10) + phase: CandidatePhase + experiment_id: str = Field(min_length=1, max_length=80) + hypothesis_id: str = Field(pattern=r"sha256:[0-9a-f]{64}") + source_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_id: str | None = Field(default=None, pattern=r"sha256:[0-9a-f]{64}") + candidate_tree: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + worktree_path: Path | None = None + job_path: Path | None = None + session_id: str | None = Field(default=None, max_length=199) + terminal_trials: int | None = Field(default=None, ge=0, le=500) + verifier_passes: int | None = Field(default=None, ge=0, le=500) + verifier_failures: int | None = Field(default=None, ge=0, le=500) + unverified_trials: int | None = Field(default=None, ge=0, le=500) + outcome_receipts: tuple[EvaluatedTaskReceipt, ...] = Field(max_length=500) + blockers: tuple[EvaluatedRunBlocker, ...] = Field(max_length=500) + evaluated_run_receipt: EvaluatedRunReceipt | None = None + next_poll_after_seconds: int | None = Field(default=None, ge=1, le=300) + error_code: CandidateErrorCode | None = None + + +@dataclass(frozen=True, slots=True) +class TraceMatchRequest: + task_id: str + session_id: str + environment: str + release: str + started_at: datetime + finished_at: datetime + + +@dataclass(frozen=True, slots=True) +class TraceMatch: + trace_id: str | None + blocker: CandidateBlockerCode | None + cost_usd: float | None = None + + def __post_init__(self) -> None: + if (self.trace_id is None) == (self.blocker is None): + raise ValueError("trace match requires exactly one result") + if self.trace_id is not None and ( + len(self.trace_id) > 256 or _IDENTIFIER_PATTERN.fullmatch(self.trace_id) is None + ): + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "trace_id") + _validate_cost(self.cost_usd) + + +def _validate_cost(cost_usd: float | None) -> None: + if cost_usd is not None and ( + not math.isfinite(cost_usd) or not 0.0 <= cost_usd <= 1_000_000.0 + ): + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "cost_usd") + + +class CandidateExperimentRunner(Protocol): + def validate( + self, + benchmark_root: Path, + harbor_executable: Path, + harbor_config: Path, + ) -> ExperimentControls: ... + + def start(self, run: ExperimentRun) -> int: ... + + def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: ... + + def cancel(self, run: ExperimentRun, process_id: int | None) -> None: ... + + +class CandidateTraceLocator(Protocol): + def locate(self, request: TraceMatchRequest) -> TraceMatch: ... + + +class CandidateOutcomeStore(Protocol): + def store(self, outcome: OutcomeEvaluation) -> OutcomeScoreSubmission: ... diff --git a/src/ofw/evolution/candidate_git.py b/src/ofw/evolution/candidate_git.py new file mode 100644 index 0000000..f1a1037 --- /dev/null +++ b/src/ofw/evolution/candidate_git.py @@ -0,0 +1,289 @@ +"""Git isolation and exact-path sealing for one harness candidate.""" + +from __future__ import annotations + +import stat +import subprocess # nosec B404 +from pathlib import Path, PurePosixPath + +from ofw.evolution.candidate import ( + CandidateCommit, + CandidateErrorCode, + CandidateFailure, + CandidateId, + CandidateTree, + CandidateWorkspace, +) +from ofw.evolution.hypothesis import HarnessHypothesis +from ofw.preparation.policy import ExperimentPolicySnapshot + +_MANAGED_PATHS = frozenset(("PROGRAM.md", "experiment_config.yaml")) +_CREDENTIAL_NAMES = frozenset( + ( + "credentials.json", + "credentials.yaml", + "credentials.yml", + "secrets.json", + "secrets.yaml", + "secrets.yml", + ) +) + + +class CandidateGitGateway: + """Create a detached candidate worktree and commit only hypothesis targets.""" + + def control_directory(self, root: Path, hypothesis_id: str) -> Path: + common = Path(_git(root, "rev-parse", "--git-common-dir")) + if not common.is_absolute(): + common = root / common + return common.resolve() / "ofw" / "candidates" / hypothesis_id.removeprefix("sha256:") + + def validate_accepted( + self, + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> None: + _validate_authority(_directory(root, "workspace_root"), policy, hypothesis) + + def prepare( + self, + accepted_root: Path, + worktree_parent: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> CandidateWorkspace: + root = _directory(accepted_root, "workspace_root") + parent = _directory(worktree_parent, "worktree_parent") + _validate_authority(root, policy, hypothesis) + worktree = parent / _worktree_name(root, policy, hypothesis) + if worktree.exists(): + raise CandidateFailure(CandidateErrorCode.WORKTREE_EXISTS, str(worktree)) + _git(root, "worktree", "add", "--detach", str(worktree), policy.initialization_commit) + return CandidateWorkspace( + accepted_root=root, + worktree_path=worktree, + source_commit=policy.initialization_commit, + ) + + def inspect( + self, + workspace: CandidateWorkspace, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> CandidateTree: + _validate_authority(workspace.accepted_root, policy, hypothesis) + _require_head(workspace.worktree_path, workspace.source_commit) + changed = _changed_paths(workspace.worktree_path) + if not changed: + raise CandidateFailure(CandidateErrorCode.EMPTY_CANDIDATE, hypothesis.id.value) + _validate_changes(workspace.worktree_path, changed, policy, hypothesis) + _git( + workspace.worktree_path, + "add", + "--all", + "--", + *(path.as_posix() for path in hypothesis.target.relative_paths), + ) + return CandidateTree( + tree_id=_git(workspace.worktree_path, "write-tree"), + changed_paths=changed, + ) + + def commit( + self, + workspace: CandidateWorkspace, + tree: CandidateTree, + candidate_id: CandidateId, + experiment_id: str, + ) -> CandidateCommit: + _require_head(workspace.worktree_path, workspace.source_commit) + if ( + _changed_paths(workspace.worktree_path) != tree.changed_paths + or not _git_succeeds(workspace.worktree_path, "diff", "--quiet") + or _git(workspace.worktree_path, "write-tree") != tree.tree_id + ): + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, experiment_id) + message = ( + "feat(ofw): record candidate execution\n\n" + f"OFW-Experiment: {experiment_id}\n" + f"OFW-Run: {candidate_id.value}" + ) + _git(workspace.worktree_path, "commit", "-m", message) + commit = _git(workspace.worktree_path, "rev-parse", "HEAD") + if _changed_paths(workspace.worktree_path): + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, experiment_id) + return CandidateCommit(commit=commit) + + +def _validate_authority( + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + _require_experiment(policy, hypothesis) + _require_source(policy, hypothesis) + _require_head(root, policy.initialization_commit) + _require_branch(root, policy) + _require_targets(policy, hypothesis) + + +def _require_experiment( + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + if hypothesis.experiment_id != policy.experiment_id: + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, hypothesis.id.value) + + +def _require_source( + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + if hypothesis.source_commit != policy.initialization_commit: + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, hypothesis.id.value) + + +def _require_branch(root: Path, policy: ExperimentPolicySnapshot) -> None: + if _git(root, "branch", "--show-current") != policy.branch_name: + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, policy.experiment_id) + + +def _require_targets( + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + if any(path not in policy.editable_paths for path in hypothesis.target.relative_paths): + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, hypothesis.id.value) + + +def _require_head(root: Path, expected: str) -> None: + if _git(root, "rev-parse", "HEAD") != expected: + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, expected) + + +def _changed_paths(root: Path) -> tuple[Path, ...]: + tracked = _git_paths(root, "diff", "--name-only", "--no-renames", "-z", "HEAD") + untracked = _git_paths(root, "ls-files", "--others", "--exclude-standard", "-z") + ignored = _git_paths(root, "ls-files", "--others", "--ignored", "--exclude-standard", "-z") + paths = tracked | untracked | ignored + return tuple(sorted(paths, key=Path.as_posix)) + + +def _git_paths(root: Path, *arguments: str) -> set[Path]: + output = _git_bytes(root, *arguments) + return {_path(value) for value in output.split(b"\0") if value} + + +def _path(value: bytes) -> Path: + try: + text = value.decode("utf-8") + except UnicodeError: + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, "encoding") from None + pure = PurePosixPath(text) + if pure.is_absolute() or pure == PurePosixPath(".") or ".." in pure.parts: + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, text) + return Path(*pure.parts) + + +def _validate_changes( + root: Path, + changed: tuple[Path, ...], + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + targets = frozenset(hypothesis.target.relative_paths) + editable = frozenset(policy.editable_paths) + for path in changed: + _validate_reserved_path(path) + if path not in editable or path not in targets: + raise CandidateFailure(CandidateErrorCode.OUT_OF_SCOPE, path.as_posix()) + _require_regular_path(root, path) + + +def _validate_reserved_path(path: Path) -> None: + text = path.as_posix() + if text in _MANAGED_PATHS or path.parts[0] == ".workspace": + raise CandidateFailure(CandidateErrorCode.MANAGED_PATH, text) + if any(_credential_name(part) for part in path.parts): + raise CandidateFailure(CandidateErrorCode.CREDENTIAL_PATH, text) + + +def _credential_name(name: str) -> bool: + lowered = name.lower() + return lowered == ".env" or lowered.startswith(".env.") or lowered in _CREDENTIAL_NAMES + + +def _require_regular_path(root: Path, relative: Path) -> None: + try: + _require_regular_parents(root, relative) + path = root / relative + metadata = path.lstat() + path.resolve(strict=True).relative_to(root.resolve(strict=True)) + except (OSError, ValueError): + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, relative.as_posix()) from None + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, relative.as_posix()) + + +def _require_regular_parents(root: Path, relative: Path) -> None: + current = root + for part in relative.parts[:-1]: + current /= part + metadata = current.lstat() + if current.is_symlink() or not stat.S_ISDIR(metadata.st_mode): + raise OSError + + +def _directory(path: Path, subject: str) -> Path: + try: + resolved = path.resolve(strict=True) + except (OSError, RuntimeError): + raise CandidateFailure(CandidateErrorCode.INVALID_WORKSPACE, subject) from None + if not resolved.is_dir(): + raise CandidateFailure(CandidateErrorCode.INVALID_WORKSPACE, subject) + return resolved + + +def _worktree_name( + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> str: + suffix = hypothesis.id.value.removeprefix("sha256:")[:12] + return f"{root.name}-ofw-{policy.experiment_id}-{suffix}" + + +def _git(root: Path, *arguments: str) -> str: + result = subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CandidateFailure(CandidateErrorCode.GIT_FAILED, arguments[0]) + return result.stdout.strip() + + +def _git_bytes(root: Path, *arguments: str) -> bytes: + result = subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + ) + if result.returncode != 0: + raise CandidateFailure(CandidateErrorCode.GIT_FAILED, arguments[0]) + return result.stdout + + +def _git_succeeds(root: Path, *arguments: str) -> bool: + return ( + subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + ).returncode + == 0 + ) diff --git a/src/ofw/evolution/candidate_langfuse.py b/src/ofw/evolution/candidate_langfuse.py new file mode 100644 index 0000000..8736f00 --- /dev/null +++ b/src/ofw/evolution/candidate_langfuse.py @@ -0,0 +1,60 @@ +"""Exact read-only Langfuse trace mapping for candidate trials.""" + +from __future__ import annotations + +from ofw.evolution.candidate import ( + CandidateBlockerCode, + TraceMatch, + TraceMatchRequest, +) +from ofw.observability.langfuse.contracts import TraceWindow +from ofw.observability.langfuse.domain import ObservationRecord, PageCursor, TraceId +from ofw.observability.langfuse.trace_query import ( + ObservationFieldGroup, + ObservationRead, + ObservationReader, +) + + +class LangfuseCandidateTraceLocator: + """Accept one trace only when the complete bounded result is unambiguous.""" + + def __init__(self, reader: ObservationReader) -> None: + self._reader = reader + + def locate(self, request: TraceMatchRequest) -> TraceMatch: + page = self._reader.read_observations( + ObservationRead( + trace_id=None, + fields=( + ObservationFieldGroup.CORE, + ObservationFieldGroup.BASIC, + ObservationFieldGroup.TRACE_CONTEXT, + ), + limit=2, + window=TraceWindow(request.started_at, request.finished_at), + session_id=request.session_id, + environment=request.environment, + release=request.release, + is_root_observation=True, + ) + ) + records = tuple(record for record in page.records if record.trace_id is not None) + trace_ids = _trace_ids(records) + return _trace_match(trace_ids, page.cursor, records) + + +def _trace_match( + trace_ids: tuple[str, ...], + cursor: PageCursor | None, + records: tuple[ObservationRecord, ...], +) -> TraceMatch: + if not trace_ids: + return TraceMatch(None, CandidateBlockerCode.TRACE_NOT_FOUND) + if cursor is not None or len(set(trace_ids)) != 1: + return TraceMatch(None, CandidateBlockerCode.TRACE_AMBIGUOUS) + return TraceMatch(TraceId(trace_ids[0]).value, None, records[0].total_cost) + + +def _trace_ids(records: tuple[ObservationRecord, ...]) -> tuple[str, ...]: + return tuple(record.trace_id.value for record in records if record.trace_id is not None) diff --git a/src/ofw/evolution/candidate_service.py b/src/ofw/evolution/candidate_service.py new file mode 100644 index 0000000..deca884 --- /dev/null +++ b/src/ofw/evolution/candidate_service.py @@ -0,0 +1,753 @@ +"""Re-entrant candidate worktree sealing, execution, and outcome recording.""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Literal + +from pydantic import Field + +from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + EvidenceReference, + OutcomeEvaluation, + RunSide, + TaskId, + VerifierId, + VerifierVerdict, +) +from ofw.evolution.candidate import ( + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateExperimentRunner, + CandidateFailure, + CandidateId, + CandidateOutcomeStore, + CandidatePhase, + CandidateStatus, + CandidateTraceLocator, + CandidateWorkspace, + TraceMatchRequest, + candidate_policy_digest, +) +from ofw.evolution.candidate_git import CandidateGitGateway +from ofw.evolution.hypothesis import HarnessHypothesis, HypothesisFailure, StrictModel +from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.observability.langfuse.domain import TraceId +from ofw.preparation.contracts import ( + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, + PreparationErrorCode, + PreparationFailure, +) +from ofw.preparation.policy import ExperimentPolicyFailure, ExperimentPolicySnapshot + + +class _CandidateState(StrictModel): + schema_version: Literal[1] = 1 + request_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + phase: CandidatePhase + source_commit: str = Field(pattern=r"[0-9a-f]{40}") + policy_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + controls_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + worktree_path: Path + candidate_id: str | None = Field(default=None, pattern=r"sha256:[0-9a-f]{64}") + candidate_tree: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + job_path: Path | None = None + log_path: Path | None = None + process_id: int | None = Field(default=None, ge=1) + started_at: datetime | None = None + deadline_at: datetime | None = None + evaluated_run_receipt: EvaluatedRunReceipt | None = None + error_code: CandidateErrorCode | None = None + + +@dataclass(frozen=True, slots=True) +class _OutcomeReduction: + receipts: tuple[EvaluatedTaskReceipt, ...] + blockers: tuple[EvaluatedRunBlocker, ...] + + +class CandidateExecutionService: + def __init__( + self, + *, + workspace: CandidateGitGateway, + hypotheses: FileHypothesisRepository, + runner: CandidateExperimentRunner, + trace_locator: CandidateTraceLocator, + outcome_store: CandidateOutcomeStore, + ) -> None: + self._workspace = workspace + self._hypotheses = hypotheses + self._runner = runner + self._trace_locator = trace_locator + self._outcome_store = outcome_store + + def execute(self, request: CandidateExecutionInput) -> CandidateExecutionObservation: + try: + return self._execute(request) + except CandidateFailure as error: + return _failure_observation(request, error) + except PreparationFailure as error: + return _failure_observation(request, _runner_failure(error)) + + def _execute(self, request: CandidateExecutionInput) -> CandidateExecutionObservation: + policy, hypothesis = self._authority(request) + control = self._workspace.control_directory( + request.workspace_root, + request.hypothesis_id, + ) + control.mkdir(parents=True, exist_ok=True) + with _candidate_lock(control): + state = _read_state(control) + if state is None: + return self._prepare(request, policy, hypothesis, control) + _validate_state(request, policy, state) + self._workspace.validate_accepted(request.workspace_root, policy, hypothesis) + if state.phase is CandidatePhase.FAILED: + return _persisted_failure_observation(request, state) + if state.phase is CandidatePhase.COMPLETE: + return _complete_observation(request, state) + if state.phase is CandidatePhase.RUNNING: + return self._poll(request, policy, control, state) + return self._launch(request, policy, hypothesis, control, state) + + def _authority( + self, + request: CandidateExecutionInput, + ) -> tuple[ExperimentPolicySnapshot, HarnessHypothesis]: + try: + policy = self._hypotheses.load_policy(request.workspace_root, request.experiment_id) + hypothesis = self._hypotheses.load(request.workspace_root, request.hypothesis_id) + except (ExperimentPolicyFailure, HypothesisFailure): + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.experiment_id) from None + return policy, hypothesis + + def _prepare( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + control: Path, + ) -> CandidateExecutionObservation: + prepared = self._workspace.prepare( + request.workspace_root, + request.worktree_parent, + policy, + hypothesis, + ) + state = _CandidateState( + request_digest=_request_digest(request), + phase=CandidatePhase.EDITING, + source_commit=prepared.source_commit, + policy_digest=candidate_policy_digest(policy), + controls_digest=policy.controls_digest, + worktree_path=prepared.worktree_path, + ) + _write_state(control, state) + return _editing_observation(request, state) + + def _launch( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + control: Path, + state: _CandidateState, + ) -> CandidateExecutionObservation: + workspace = _git_workspace(request, state) + tree = self._workspace.inspect(workspace, policy, hypothesis) + controls = self._validated_controls(request, policy) + candidate_id = CandidateId.build( + policy_digest=state.policy_digest, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_tree=tree.tree_id, + controls_digest=state.controls_digest, + ) + committed = self._workspace.commit( + workspace, + tree, + candidate_id, + request.experiment_id, + ) + run = _new_run(request, control, workspace, controls, candidate_id, committed.commit) + started_at = datetime.now(UTC) + running = _running_state( + state, + candidate_id, + tree.tree_id, + committed.commit, + run, + started_at, + None, + policy.max_baseline_seconds, + ) + _write_state(control, running) + try: + process_id = self._runner.start(run) + except PreparationFailure as error: + failure = _runner_failure(error) + failed = _failed_state(running, failure.code) + _write_state(control, failed) + return _persisted_failure_observation(request, failed) + launched = _running_state( + state, + candidate_id, + tree.tree_id, + committed.commit, + run, + started_at, + process_id, + policy.max_baseline_seconds, + ) + _write_state(control, launched) + return _running_observation(request, launched) + + def _poll( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + control: Path, + state: _CandidateState, + ) -> CandidateExecutionObservation: + controls = self._validated_controls(request, policy) + run = _run_from_state(request, state, controls) + summary = self._runner.summarize(run) + if summary is None: + if _deadline_expired(state): + self._runner.cancel(run, state.process_id) + failed = _failed_state(state, CandidateErrorCode.CANDIDATE_TIMEOUT) + _write_state(control, failed) + return _persisted_failure_observation(request, failed) + return _running_observation(request, state) + reduction = _record_outcomes( + summary, + run, + controls, + self._trace_locator, + self._outcome_store, + ) + evaluated = _evaluated_receipt(state, run, controls, reduction) + complete = _complete_state(state, evaluated) + _write_state(control, complete) + return _complete_observation(request, complete) + + def _validated_controls( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + ) -> ExperimentControls: + actual = self._runner.validate( + request.benchmark_root, + request.harbor_executable, + request.harbor_config, + ) + if actual != _policy_controls(policy): + raise CandidateFailure(CandidateErrorCode.CONTROLS_DRIFT, policy.experiment_id) + return actual + + +def _record_outcomes( + summary: ExperimentSummary, + run: ExperimentRun, + controls: ExperimentControls, + trace_locator: CandidateTraceLocator, + outcome_store: CandidateOutcomeStore, +) -> _OutcomeReduction: + receipts: list[EvaluatedTaskReceipt] = [] + blockers: list[EvaluatedRunBlocker] = [] + for trial in summary.trials: + result = _authoritative_result(trial) + if isinstance(result, EvaluatedRunBlocker): + blockers.append(result) + continue + match = trace_locator.locate(_trace_request(trial, run, controls)) + if match.trace_id is None: + blockers.append(_trace_blocker(trial, match.blocker)) + continue + outcome = _outcome(trial, controls, match.trace_id, result) + try: + submission = outcome_store.store(outcome) + except Exception: + raise CandidateFailure( + CandidateErrorCode.OUTCOME_STORE_FAILED, + trial.task_id, + ) from None + receipts.append( + EvaluatedTaskReceipt( + task_id=trial.task_id, + trace_id=match.trace_id, + score_id=submission.score_id.value, + verdict=result[0], + verifier_id=outcome.verifier_id.value, + normalized_score=result[1], + cost_usd=match.cost_usd, + latency_seconds=trial.latency_seconds, + ) + ) + return _OutcomeReduction(tuple(receipts), tuple(blockers)) + + +def _evaluated_receipt( + state: _CandidateState, + run: ExperimentRun, + controls: ExperimentControls, + reduction: _OutcomeReduction, +) -> EvaluatedRunReceipt: + if state.candidate_commit is None or state.candidate_tree is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, run.run_id) + return EvaluatedRunReceipt.build( + run_id=run.run_id, + side=RunSide.CANDIDATE, + policy_digest=state.policy_digest, + controls_digest=state.controls_digest, + evaluated_commit=state.candidate_commit, + evaluated_tree=state.candidate_tree, + task_ids=controls.task_ids, + outcome_receipts=reduction.receipts, + blockers=reduction.blockers, + ) + + +def _authoritative_result( + trial: ExperimentTrial, +) -> tuple[VerifierVerdict, float | None] | EvaluatedRunBlocker: + if trial.exception: + return _blocker(trial, CandidateBlockerCode.UNVERIFIED, "agent_exception") + reward = _reward_result(trial) + if reward is not None: + return reward + return _verdict_result(trial) + + +def _reward_result( + trial: ExperimentTrial, +) -> tuple[VerifierVerdict, float] | EvaluatedRunBlocker | None: + if trial.reward == 1.0: + return VerifierVerdict.PASS, 1.0 + if trial.reward == 0.0: + return VerifierVerdict.FAIL, 0.0 + if trial.reward is not None: + return _blocker(trial, CandidateBlockerCode.UNSUPPORTED_REWARD, str(trial.reward)) + return None + + +def _verdict_result( + trial: ExperimentTrial, +) -> tuple[VerifierVerdict, None] | EvaluatedRunBlocker: + if trial.verdict in (VerifierVerdict.ABSTAIN.value, VerifierVerdict.ERROR.value): + return VerifierVerdict(trial.verdict), None + return _blocker(trial, CandidateBlockerCode.UNVERIFIED, "missing_verifier_result") + + +def _trace_request( + trial: ExperimentTrial, + run: ExperimentRun, + controls: ExperimentControls, +) -> TraceMatchRequest: + return TraceMatchRequest( + task_id=trial.task_id, + session_id=run.session_id, + environment=controls.environment, + release=run.release, + started_at=trial.started_at, + finished_at=trial.finished_at, + ) + + +def _trace_blocker( + trial: ExperimentTrial, + code: CandidateBlockerCode | None, +) -> EvaluatedRunBlocker: + if code is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, trial.task_id) + return _blocker(trial, code, "trace_mapping") + + +def _blocker( + trial: ExperimentTrial, + code: CandidateBlockerCode, + subject: str, +) -> EvaluatedRunBlocker: + return EvaluatedRunBlocker(task_id=trial.task_id, code=code.value, subject=subject) + + +def _outcome( + trial: ExperimentTrial, + controls: ExperimentControls, + trace_id: str, + result: tuple[VerifierVerdict, float | None], +) -> OutcomeEvaluation: + verdict, score = result + return OutcomeEvaluation( + trace_id=TraceId(trace_id), + task_id=TaskId(trial.task_id), + verifier_id=VerifierId(f"{controls.verifier}@{trial.task_checksum}"), + evaluated_at=trial.evaluated_at, + verdict=verdict, + score=score, + evidence=tuple(EvidenceReference(value) for value in trial.evidence), + ) + + +def _policy_controls(policy: ExperimentPolicySnapshot) -> ExperimentControls: + return ExperimentControls( + model=policy.model, + task_ids=policy.task_ids, + benchmark_config_digest=policy.benchmark_config_digest, + verifier=policy.verifier, + environment=policy.environment, + concurrency=policy.concurrency, + max_retries=policy.max_retries, + ) + + +def _new_run( + request: CandidateExecutionInput, + control: Path, + workspace: CandidateWorkspace, + controls: ExperimentControls, + candidate_id: CandidateId, + candidate_commit: str, +) -> ExperimentRun: + run_id = f"ofw-candidate-{candidate_id.value.removeprefix('sha256:')[:24]}" + return ExperimentRun( + run_id=run_id, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=request.benchmark_root / request.harbor_config, + job_path=request.benchmark_root / "jobs" / run_id, + log_path=control / "candidate.log", + source_root=workspace.worktree_path, + release=candidate_commit, + session_id=candidate_id.value, + controls=controls, + ) + + +def _run_from_state( + request: CandidateExecutionInput, + state: _CandidateState, + controls: ExperimentControls, +) -> ExperimentRun: + candidate_id = state.candidate_id + candidate_commit = state.candidate_commit + job_path = state.job_path + log_path = state.log_path + if candidate_id is None or candidate_commit is None or job_path is None or log_path is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, request.hypothesis_id) + return ExperimentRun( + run_id=job_path.name, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=request.benchmark_root / request.harbor_config, + job_path=job_path, + log_path=log_path, + source_root=state.worktree_path, + release=candidate_commit, + session_id=candidate_id, + controls=controls, + ) + + +def _git_workspace( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateWorkspace: + return CandidateWorkspace( + accepted_root=request.workspace_root.resolve(), + worktree_path=state.worktree_path, + source_commit=state.source_commit, + ) + + +def _running_state( + previous: _CandidateState, + candidate_id: CandidateId, + tree: str, + commit: str, + run: ExperimentRun, + started_at: datetime, + process_id: int | None, + timeout_seconds: int, +) -> _CandidateState: + return _CandidateState( + request_digest=previous.request_digest, + phase=CandidatePhase.RUNNING, + source_commit=previous.source_commit, + policy_digest=previous.policy_digest, + controls_digest=previous.controls_digest, + worktree_path=previous.worktree_path, + candidate_id=candidate_id.value, + candidate_tree=tree, + candidate_commit=commit, + job_path=run.job_path, + log_path=run.log_path, + process_id=process_id, + started_at=started_at, + deadline_at=started_at + timedelta(seconds=timeout_seconds), + evaluated_run_receipt=None, + ) + + +def _complete_state( + state: _CandidateState, + evaluated: EvaluatedRunReceipt, +) -> _CandidateState: + return _CandidateState( + request_digest=state.request_digest, + phase=CandidatePhase.COMPLETE, + source_commit=state.source_commit, + policy_digest=state.policy_digest, + controls_digest=state.controls_digest, + worktree_path=state.worktree_path, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + job_path=state.job_path, + log_path=state.log_path, + process_id=state.process_id, + started_at=state.started_at, + deadline_at=state.deadline_at, + evaluated_run_receipt=evaluated, + ) + + +def _failed_state(state: _CandidateState, code: CandidateErrorCode) -> _CandidateState: + return _CandidateState( + request_digest=state.request_digest, + phase=CandidatePhase.FAILED, + source_commit=state.source_commit, + policy_digest=state.policy_digest, + controls_digest=state.controls_digest, + worktree_path=state.worktree_path, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + job_path=state.job_path, + log_path=state.log_path, + process_id=state.process_id, + started_at=state.started_at, + deadline_at=state.deadline_at, + evaluated_run_receipt=state.evaluated_run_receipt, + error_code=code, + ) + + +def _validate_state( + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + state: _CandidateState, +) -> None: + if state.request_digest != _request_digest(request): + raise CandidateFailure(CandidateErrorCode.REQUEST_CONFLICT, request.hypothesis_id) + if state.policy_digest != candidate_policy_digest(policy): + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.experiment_id) + if state.controls_digest != policy.controls_digest: + raise CandidateFailure(CandidateErrorCode.CONTROLS_DRIFT, request.experiment_id) + + +def _deadline_expired(state: _CandidateState) -> bool: + return state.deadline_at is not None and datetime.now(UTC) > state.deadline_at + + +def _editing_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + return CandidateExecutionObservation( + status=CandidateStatus.WARNING, + summary="The isolated candidate worktree is ready for the hypothesis edit.", + next_actions=( + "Edit only the exact hypothesis targets, then call execute_candidate again.", + ), + artifacts=(str(state.worktree_path),), + phase=CandidatePhase.EDITING, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + worktree_path=state.worktree_path, + outcome_receipts=(), + blockers=(), + ) + + +def _running_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + return CandidateExecutionObservation( + status=CandidateStatus.WARNING, + summary="The deterministic candidate Harbor run is still running.", + next_actions=("Poll execute_candidate with the identical request.",), + artifacts=_artifacts(state), + phase=CandidatePhase.RUNNING, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + worktree_path=state.worktree_path, + job_path=state.job_path, + session_id=state.candidate_id, + outcome_receipts=(), + blockers=(), + next_poll_after_seconds=30, + ) + + +def _complete_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + receipt = state.evaluated_run_receipt + if receipt is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, request.hypothesis_id) + passes = sum(item.verdict is VerifierVerdict.PASS for item in receipt.outcome_receipts) + failures = sum(item.verdict is VerifierVerdict.FAIL for item in receipt.outcome_receipts) + terminal = len(receipt.outcome_receipts) + len(receipt.blockers) + return CandidateExecutionObservation( + status=CandidateStatus.WARNING if receipt.blockers else CandidateStatus.SUCCESS, + summary="The candidate run is complete with authoritative outcome receipts.", + next_actions=("Retain the candidate and outcome receipts for the admission gate.",), + artifacts=_artifacts(state), + phase=CandidatePhase.COMPLETE, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + worktree_path=state.worktree_path, + job_path=state.job_path, + session_id=state.candidate_id, + terminal_trials=terminal, + verifier_passes=passes, + verifier_failures=failures, + unverified_trials=len(receipt.blockers), + outcome_receipts=receipt.outcome_receipts, + blockers=receipt.blockers, + evaluated_run_receipt=receipt, + ) + + +def _failure_observation( + request: CandidateExecutionInput, + error: CandidateFailure, +) -> CandidateExecutionObservation: + return CandidateExecutionObservation( + status=CandidateStatus.ERROR, + summary=f"Candidate execution stopped: {error.code.value}.", + next_actions=("Correct the typed boundary failure without forcing Git state.",), + artifacts=(), + phase=CandidatePhase.FAILED, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + outcome_receipts=(), + blockers=(), + error_code=error.code, + ) + + +def _persisted_failure_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + code = state.error_code or CandidateErrorCode.INVALID_RESULT + return CandidateExecutionObservation( + status=CandidateStatus.ERROR, + summary=f"Candidate execution stopped: {code.value}.", + next_actions=("Retain the terminal candidate failure receipt.",), + artifacts=_artifacts(state), + phase=CandidatePhase.FAILED, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + worktree_path=state.worktree_path, + job_path=state.job_path, + session_id=state.candidate_id, + outcome_receipts=(), + blockers=(), + evaluated_run_receipt=state.evaluated_run_receipt, + error_code=code, + ) + + +def _artifacts(state: _CandidateState) -> tuple[str, ...]: + values = ( + state.candidate_id, + state.candidate_commit, + None if state.job_path is None else str(state.job_path), + ) + return tuple(value for value in values if value is not None) + + +def _request_digest(request: CandidateExecutionInput) -> str: + digest = hashlib.sha256(request.model_dump_json().encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +@contextmanager +def _candidate_lock(control: Path) -> Iterator[None]: + lock = control / ".lock" + try: + lock.mkdir() + except FileExistsError: + raise CandidateFailure(CandidateErrorCode.REQUEST_CONFLICT, control.name) from None + try: + yield + finally: + lock.rmdir() + + +def _read_state(control: Path) -> _CandidateState | None: + path = control / "state.json" + if not path.exists(): + return None + try: + return _CandidateState.model_validate_json(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "state.json") from None + + +def _write_state(control: Path, state: _CandidateState) -> None: + path = control / "state.json" + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=control, + delete=False, + ) as temporary: + temporary.write(state.model_dump_json(indent=2) + "\n") + temporary_path = Path(temporary.name) + os.replace(temporary_path, path) + + +def _runner_failure(error: PreparationFailure) -> CandidateFailure: + if error.code is PreparationErrorCode.MISSING_ENVIRONMENT: + code = CandidateErrorCode.MISSING_ENVIRONMENT + elif error.code is PreparationErrorCode.LAUNCH_FAILED: + code = CandidateErrorCode.LAUNCH_FAILED + else: + code = CandidateErrorCode.INVALID_RESULT + return CandidateFailure(code, error.subject) diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py new file mode 100644 index 0000000..3b60975 --- /dev/null +++ b/src/ofw/evolution/hypothesis.py @@ -0,0 +1,612 @@ +"""Immutable evidence-backed hypothesis contract and recording service.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Literal, Protocol + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from ofw.contracts import ComponentKind +from ofw.evaluation.failure_curation import FailureCurationArtifact, FailureGroupArtifact +from ofw.evaluation.failure_patterns import ( + FailurePatternMiningError, + FailurePatternMiningObservation, + FailurePatternMiningService, + MineFailurePatternsInput, +) +from ofw.preparation.contracts import contained_relative_path +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, +) + +_ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +_PATTERN_ID_PATTERN = r"^sha256:[0-9a-f]{64}$" +_HYPOTHESIS_ID_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_EXPERIMENT_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" +_TASK_ID_PATTERN = r"^[^\x00]+$" +HypothesisTextValue = Annotated[str, Field(min_length=1, max_length=4000)] +ArtifactIdsValue = Annotated[ + tuple[Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)], ...], + Field(strict=False, min_length=1, max_length=50), +] +RelativePathsValue = Annotated[ + tuple[Annotated[Path, Field(strict=False)], ...], + Field(strict=False, min_length=1, max_length=50), +] +TaskIdsValue = Annotated[ + tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], + Field(strict=False, min_length=1, max_length=50), +] +RiskTaskIdsValue = Annotated[ + tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], + Field(strict=False, max_length=50), +] + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class FailurePatternReferenceInput(StrictModel): + pattern_id: str = Field(pattern=_PATTERN_ID_PATTERN) + diagnosis_artifact_ids: ArtifactIdsValue + + @field_validator("diagnosis_artifact_ids") + @classmethod + def validate_unique_artifacts(cls, values: tuple[str, ...]) -> tuple[str, ...]: + if len(set(values)) != len(values): + raise ValueError("diagnosis_artifact_ids must be unique") + return values + + +class HarnessChangeTargetInput(StrictModel): + component_kind: Annotated[ComponentKind, Field(strict=False)] + relative_paths: RelativePathsValue + + @field_validator("relative_paths") + @classmethod + def validate_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: + normalized = tuple(contained_relative_path(value, "relative_paths") for value in values) + if len(set(normalized)) != len(normalized): + raise ValueError("relative_paths must be unique") + return normalized + + +class RecordHypothesisInput(StrictModel): + workspace_root: Path = Field(strict=False) + experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) + source_commit: str = Field(pattern=_COMMIT_PATTERN) + curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + predicted_task_ids: TaskIdsValue + at_risk_task_ids: RiskTaskIdsValue = () + patterns: tuple[FailurePatternReferenceInput, ...] = Field( + strict=False, + min_length=1, + max_length=50, + ) + statement: HypothesisTextValue + rationale: HypothesisTextValue + target: HarnessChangeTargetInput + expected_effect: HypothesisTextValue + regression_risks: tuple[HypothesisTextValue, ...] = Field( + strict=False, + max_length=10, + ) + + @field_validator("workspace_root") + @classmethod + def validate_workspace_root(cls, value: Path) -> Path: + if not value.is_absolute(): + raise ValueError("workspace_root must be absolute") + return value + + @field_validator("statement", "rationale", "expected_effect") + @classmethod + def normalize_text(cls, value: str) -> str: + return _normalized_text(value) + + @field_validator("regression_risks") + @classmethod + def validate_risks(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(_normalized_text(value) for value in values) + if len(set(normalized)) != len(normalized): + raise ValueError("regression_risks must be unique") + return normalized + + @model_validator(mode="after") + def validate_evidence_bound(self) -> RecordHypothesisInput: + pattern_ids = tuple(pattern.pattern_id for pattern in self.patterns) + artifact_ids = tuple( + artifact_id + for pattern in self.patterns + for artifact_id in pattern.diagnosis_artifact_ids + ) + _require_unique_patterns(pattern_ids) + _require_bounded_unique_artifacts(artifact_ids) + _require_disjoint_task_predictions(self.predicted_task_ids, self.at_risk_task_ids) + return self + + +@dataclass(frozen=True, slots=True) +class HypothesisId: + value: str + + def __post_init__(self) -> None: + if _HYPOTHESIS_ID_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid hypothesis id") + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class FailurePatternReference: + pattern_id: str + diagnosis_artifact_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class HarnessChangeTarget: + component_kind: ComponentKind + relative_paths: tuple[Path, ...] + + +@dataclass(frozen=True, slots=True) +class HarnessHypothesis: + id: HypothesisId + experiment_id: str + source_commit: str + curation_id: str + curation_group_id: str + predicted_task_ids: tuple[str, ...] + at_risk_task_ids: tuple[str, ...] + patterns: tuple[FailurePatternReference, ...] + statement: str + rationale: str + target: HarnessChangeTarget + expected_effect: str + regression_risks: tuple[str, ...] + + +class _HypothesisContent(StrictModel): + schema_version: Literal[1] = 1 + experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) + source_commit: str = Field(pattern=_COMMIT_PATTERN) + curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + predicted_task_ids: TaskIdsValue + at_risk_task_ids: RiskTaskIdsValue = () + patterns: tuple[FailurePatternReferenceInput, ...] = Field(min_length=1, max_length=50) + statement: HypothesisTextValue + rationale: HypothesisTextValue + target: HarnessChangeTargetInput + expected_effect: HypothesisTextValue + regression_risks: tuple[HypothesisTextValue, ...] = Field(max_length=10) + + +class HypothesisArtifact(_HypothesisContent): + hypothesis_id: str = Field(pattern=_PATTERN_ID_PATTERN) + + @classmethod + def from_hypothesis(cls, hypothesis: HarnessHypothesis) -> HypothesisArtifact: + return cls( + hypothesis_id=hypothesis.id.value, + experiment_id=hypothesis.experiment_id, + source_commit=hypothesis.source_commit, + curation_id=hypothesis.curation_id, + curation_group_id=hypothesis.curation_group_id, + predicted_task_ids=hypothesis.predicted_task_ids, + at_risk_task_ids=hypothesis.at_risk_task_ids, + patterns=tuple( + FailurePatternReferenceInput( + pattern_id=pattern.pattern_id, + diagnosis_artifact_ids=pattern.diagnosis_artifact_ids, + ) + for pattern in hypothesis.patterns + ), + statement=hypothesis.statement, + rationale=hypothesis.rationale, + target=HarnessChangeTargetInput( + component_kind=hypothesis.target.component_kind, + relative_paths=hypothesis.target.relative_paths, + ), + expected_effect=hypothesis.expected_effect, + regression_risks=hypothesis.regression_risks, + ) + + def to_hypothesis(self) -> HarnessHypothesis: + return HarnessHypothesis( + id=HypothesisId(self.hypothesis_id), + experiment_id=self.experiment_id, + source_commit=self.source_commit, + curation_id=self.curation_id, + curation_group_id=self.curation_group_id, + predicted_task_ids=self.predicted_task_ids, + at_risk_task_ids=self.at_risk_task_ids, + patterns=tuple( + FailurePatternReference( + pattern.pattern_id, + pattern.diagnosis_artifact_ids, + ) + for pattern in self.patterns + ), + statement=self.statement, + rationale=self.rationale, + target=HarnessChangeTarget( + self.target.component_kind, + self.target.relative_paths, + ), + expected_effect=self.expected_effect, + regression_risks=self.regression_risks, + ) + + def recomputed_id(self) -> HypothesisId: + return _hypothesis_id( + _HypothesisContent( + experiment_id=self.experiment_id, + source_commit=self.source_commit, + curation_id=self.curation_id, + curation_group_id=self.curation_group_id, + predicted_task_ids=self.predicted_task_ids, + at_risk_task_ids=self.at_risk_task_ids, + patterns=self.patterns, + statement=self.statement, + rationale=self.rationale, + target=self.target, + expected_effect=self.expected_effect, + regression_risks=self.regression_risks, + ) + ) + + +class HypothesisStatus(StrEnum): + SUCCESS = "success" + + +class HypothesisObservation(StrictModel): + status: HypothesisStatus + summary: str = Field(min_length=1, max_length=256) + next_actions: tuple[str, ...] = Field(max_length=2) + artifacts: tuple[str, ...] = Field(min_length=2, max_length=2) + hypothesis_id: str = Field(pattern=_PATTERN_ID_PATTERN) + experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) + source_commit: str = Field(pattern=_COMMIT_PATTERN) + curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + predicted_task_ids: TaskIdsValue + at_risk_task_ids: RiskTaskIdsValue = () + relative_path: Path + pattern_count: int = Field(strict=True, ge=1, le=50) + diagnosis_count: int = Field(strict=True, ge=1, le=50) + target_paths: RelativePathsValue + + +class HypothesisErrorCode(StrEnum): + POLICY_SNAPSHOT_REQUIRED = "policy_snapshot_required" + POLICY_INVALID = "policy_invalid" + STALE_POLICY = "stale_policy" + STALE_COMMIT = "stale_commit" + DIRTY_WORKSPACE = "dirty_workspace" + CURATION_NOT_FOUND = "curation_not_found" + CURATION_INVALID = "curation_invalid" + CURATION_GROUP_NOT_FOUND = "curation_group_not_found" + CURATION_EVIDENCE_MISMATCH = "curation_evidence_mismatch" + PATTERN_EVIDENCE_MISMATCH = "pattern_evidence_mismatch" + INCONCLUSIVE_EVIDENCE = "inconclusive_evidence" + TARGET_NOT_EDITABLE = "target_not_editable" + INVALID_TARGET = "invalid_target" + HYPOTHESIS_CONFLICT = "hypothesis_conflict" + HYPOTHESIS_TOO_LARGE = "hypothesis_too_large" + WRITE_FAILED = "write_failed" + + +class HypothesisFailure(Exception): + """Typed sanitized hypothesis rejection.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: HypothesisErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +class HypothesisGateway(Protocol): + def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: ... + + def load_curation(self, root: Path, curation_id: str) -> FailureCurationArtifact: ... + + def validate_workspace( + self, + root: Path, + policy: ExperimentPolicySnapshot, + source_commit: str, + paths: tuple[Path, ...], + ) -> None: ... + + def store(self, root: Path, hypothesis: HarnessHypothesis) -> Path: ... + + +@dataclass(frozen=True, slots=True) +class HypothesisService: + pattern_miner: FailurePatternMiningService + repository: HypothesisGateway + + def record(self, request: RecordHypothesisInput) -> HypothesisObservation: + policy = _load_policy(self.repository, request) + _validate_policy(request, policy) + paths = tuple(sorted(request.target.relative_paths, key=Path.as_posix)) + _validate_editable_paths(paths, policy.editable_paths) + self.repository.validate_workspace( + request.workspace_root, + policy, + request.source_commit, + paths, + ) + group = _load_curation_group(self.repository, request) + patterns = _canonical_patterns(request.patterns) + _validate_curated_evidence(request, group, patterns) + mined = _mine_patterns(self.pattern_miner, request.workspace_root, patterns) + _validate_mined_patterns(patterns, mined) + hypothesis = _hypothesis(request, patterns, paths) + relative_path = self.repository.store(request.workspace_root, hypothesis) + return _observation(hypothesis, relative_path) + + +def _load_policy( + repository: HypothesisGateway, + request: RecordHypothesisInput, +) -> ExperimentPolicySnapshot: + try: + return repository.load_policy(request.workspace_root, request.experiment_id) + except ExperimentPolicyFailure as error: + code = ( + HypothesisErrorCode.POLICY_SNAPSHOT_REQUIRED + if error.code is ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED + else HypothesisErrorCode.POLICY_INVALID + ) + raise HypothesisFailure(code, request.experiment_id) from None + + +def _load_curation_group( + repository: HypothesisGateway, + request: RecordHypothesisInput, +) -> FailureGroupArtifact: + curation = repository.load_curation(request.workspace_root, request.curation_id) + group = next( + (item for item in curation.groups if item.group_id == request.curation_group_id), + None, + ) + if group is None: + raise HypothesisFailure( + HypothesisErrorCode.CURATION_GROUP_NOT_FOUND, + request.curation_group_id, + ) + return group + + +def _validate_policy( + request: RecordHypothesisInput, + policy: ExperimentPolicySnapshot, +) -> None: + if request.source_commit != policy.initialization_commit: + raise HypothesisFailure(HypothesisErrorCode.STALE_COMMIT, request.experiment_id) + + +def _validate_editable_paths(paths: tuple[Path, ...], editable: tuple[Path, ...]) -> None: + for path in paths: + if path not in editable: + raise HypothesisFailure(HypothesisErrorCode.TARGET_NOT_EDITABLE, path.as_posix()) + + +def _canonical_patterns( + patterns: tuple[FailurePatternReferenceInput, ...], +) -> tuple[FailurePatternReference, ...]: + return tuple( + FailurePatternReference( + pattern.pattern_id, + tuple(sorted(pattern.diagnosis_artifact_ids)), + ) + for pattern in sorted(patterns, key=_pattern_reference_id) + ) + + +def _validate_curated_evidence( + request: RecordHypothesisInput, + group: FailureGroupArtifact, + patterns: tuple[FailurePatternReference, ...], +) -> None: + if _declared_artifact_ids(patterns) != _curated_artifact_ids(group): + raise HypothesisFailure( + HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH, + request.curation_group_id, + ) + if request.target.component_kind is not group.target_component: + raise HypothesisFailure( + HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH, + request.curation_group_id, + ) + + +def _declared_artifact_ids( + patterns: tuple[FailurePatternReference, ...], +) -> tuple[str, ...]: + return tuple( + sorted( + artifact_id + for pattern in patterns + for artifact_id in pattern.diagnosis_artifact_ids + ) + ) + + +def _curated_artifact_ids(group: FailureGroupArtifact) -> tuple[str, ...]: + return tuple(sorted(member.artifact_id for member in group.members)) + + +def _require_disjoint_task_predictions( + predicted: tuple[str, ...], + at_risk: tuple[str, ...], +) -> None: + if set(predicted) & set(at_risk): + raise ValueError("predicted_task_ids and at_risk_task_ids must be disjoint") + + +def _pattern_reference_id(pattern: FailurePatternReferenceInput) -> str: + return pattern.pattern_id + + +def _mine_patterns( + service: FailurePatternMiningService, + root: Path, + patterns: tuple[FailurePatternReference, ...], +) -> FailurePatternMiningObservation: + artifact_ids = tuple( + sorted( + artifact_id for pattern in patterns for artifact_id in pattern.diagnosis_artifact_ids + ) + ) + try: + request = MineFailurePatternsInput(workspace_root=root, artifact_ids=artifact_ids) + return service.mine(request) + except FailurePatternMiningError as error: + raise HypothesisFailure( + HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH, + error.subject, + ) from None + + +def _validate_mined_patterns( + declared: tuple[FailurePatternReference, ...], + mined: FailurePatternMiningObservation, +) -> None: + if mined.inconclusive_artifact_ids: + raise HypothesisFailure( + HypothesisErrorCode.INCONCLUSIVE_EVIDENCE, + mined.inconclusive_artifact_ids[0], + ) + if len(declared) != len(mined.patterns): + raise HypothesisFailure(HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH, "patterns") + for reference in declared: + _validate_pattern_reference(reference, mined) + + +def _validate_pattern_reference( + reference: FailurePatternReference, + mined: FailurePatternMiningObservation, +) -> None: + matching = tuple(item for item in mined.patterns if item.pattern_id == reference.pattern_id) + if len(matching) != 1 or matching[0].artifact_ids != reference.diagnosis_artifact_ids: + raise HypothesisFailure( + HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH, + reference.pattern_id, + ) + + +def _hypothesis( + request: RecordHypothesisInput, + patterns: tuple[FailurePatternReference, ...], + paths: tuple[Path, ...], +) -> HarnessHypothesis: + target = HarnessChangeTarget(request.target.component_kind, paths) + risks = tuple(sorted(request.regression_risks)) + content = _content(request, patterns, target, risks) + hypothesis_id = _hypothesis_id(content) + return HarnessHypothesis( + hypothesis_id, + request.experiment_id, + request.source_commit, + request.curation_id, + request.curation_group_id, + request.predicted_task_ids, + request.at_risk_task_ids, + patterns, + request.statement, + request.rationale, + target, + request.expected_effect, + risks, + ) + + +def _hypothesis_id(content: _HypothesisContent) -> HypothesisId: + digest = hashlib.sha256(content.model_dump_json().encode("utf-8")).hexdigest() + return HypothesisId(f"sha256:{digest}") + + +def _content( + request: RecordHypothesisInput, + patterns: tuple[FailurePatternReference, ...], + target: HarnessChangeTarget, + risks: tuple[str, ...], +) -> _HypothesisContent: + return _HypothesisContent( + experiment_id=request.experiment_id, + source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, + patterns=tuple( + FailurePatternReferenceInput( + pattern_id=pattern.pattern_id, + diagnosis_artifact_ids=pattern.diagnosis_artifact_ids, + ) + for pattern in patterns + ), + statement=request.statement, + rationale=request.rationale, + target=HarnessChangeTargetInput( + component_kind=target.component_kind, + relative_paths=target.relative_paths, + ), + expected_effect=request.expected_effect, + regression_risks=risks, + ) + + +def _observation(hypothesis: HarnessHypothesis, relative_path: Path) -> HypothesisObservation: + diagnosis_count = sum(len(pattern.diagnosis_artifact_ids) for pattern in hypothesis.patterns) + return HypothesisObservation( + status=HypothesisStatus.SUCCESS, + summary="Recorded one evidence-backed hypothesis for the prepared experiment.", + next_actions=("Stop before candidate editing and retain this hypothesis receipt.",), + artifacts=(str(relative_path), hypothesis.id.value), + hypothesis_id=hypothesis.id.value, + experiment_id=hypothesis.experiment_id, + source_commit=hypothesis.source_commit, + curation_id=hypothesis.curation_id, + curation_group_id=hypothesis.curation_group_id, + predicted_task_ids=hypothesis.predicted_task_ids, + at_risk_task_ids=hypothesis.at_risk_task_ids, + relative_path=relative_path, + pattern_count=len(hypothesis.patterns), + diagnosis_count=diagnosis_count, + target_paths=hypothesis.target.relative_paths, + ) + + +def _require_unique_patterns(pattern_ids: tuple[str, ...]) -> None: + if len(set(pattern_ids)) != len(pattern_ids): + raise ValueError("pattern_ids must be unique") + + +def _require_bounded_unique_artifacts(artifact_ids: tuple[str, ...]) -> None: + if len(artifact_ids) > 50 or len(set(artifact_ids)) != len(artifact_ids): + raise ValueError("diagnosis artifacts must be globally unique and bounded") + + +def _normalized_text(value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("hypothesis text must not be blank") + return normalized diff --git a/src/ofw/evolution/hypothesis_repository.py b/src/ofw/evolution/hypothesis_repository.py new file mode 100644 index 0000000..150d61e --- /dev/null +++ b/src/ofw/evolution/hypothesis_repository.py @@ -0,0 +1,192 @@ +"""Git and safe-file repository for prepared-experiment hypotheses.""" + +from __future__ import annotations + +import stat +import subprocess # nosec B404 +from pathlib import Path + +from pydantic import ValidationError + +from ofw.evaluation.failure_curation import FailureCurationArtifact +from ofw.evolution.hypothesis import ( + HarnessHypothesis, + HypothesisArtifact, + HypothesisErrorCode, + HypothesisFailure, +) +from ofw.preparation.policy import ( + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_child_directory, + open_directory_chain, + publish_idempotent, + read_bounded, +) + +_HYPOTHESIS_LIMIT_BYTES = 64 * 1024 +_WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") + + +class FileHypothesisRepository: + """Validate one prepared Git worktree and store immutable hypothesis JSON.""" + + def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: + return FileExperimentPolicyRepository().load(root, experiment_id) + + def load_curation(self, root: Path, curation_id: str) -> FailureCurationArtifact: + prepared_root = _prepared_root(root) + try: + with open_directory_chain( + prepared_root, + (".workspace", "failure-curations"), + create=False, + ) as directory: + content = read_bounded( + directory, + f"{curation_id}.json", + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=curation_id, + ) + artifact = FailureCurationArtifact.model_validate_json(content) + artifact.require_valid_identity() + except FileNotFoundError: + raise HypothesisFailure(HypothesisErrorCode.CURATION_NOT_FOUND, curation_id) from None + except (SafeFileFailure, ValidationError, ValueError): + raise HypothesisFailure(HypothesisErrorCode.CURATION_INVALID, curation_id) from None + if artifact.curation_id != curation_id: + raise HypothesisFailure(HypothesisErrorCode.CURATION_INVALID, curation_id) + return artifact + + def load(self, root: Path, hypothesis_id: str) -> HarnessHypothesis: + prepared_root = _prepared_root(root) + try: + with open_directory_chain( + prepared_root, + (".workspace", "hypotheses"), + create=False, + ) as directory: + content = read_bounded( + directory, + f"{hypothesis_id}.json", + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=hypothesis_id, + ) + artifact = HypothesisArtifact.model_validate_json(content) + except (FileNotFoundError, SafeFileFailure, ValidationError, ValueError, OSError): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, hypothesis_id) from None + if ( + artifact.hypothesis_id != hypothesis_id + or artifact.recomputed_id().value != hypothesis_id + ): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, hypothesis_id) + return artifact.to_hypothesis() + + def validate_workspace( + self, + root: Path, + policy: ExperimentPolicySnapshot, + source_commit: str, + paths: tuple[Path, ...], + ) -> None: + prepared_root = _prepared_root(root) + if _git(prepared_root, "rev-parse", "HEAD") != source_commit: + raise HypothesisFailure(HypothesisErrorCode.STALE_COMMIT, policy.experiment_id) + if _git(prepared_root, "branch", "--show-current") != policy.branch_name: + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, policy.experiment_id) + if _git(prepared_root, "status", "--porcelain=v1"): + raise HypothesisFailure(HypothesisErrorCode.DIRTY_WORKSPACE, policy.experiment_id) + for path in paths: + _require_regular_target(prepared_root, path) + + def store(self, root: Path, hypothesis: HarnessHypothesis) -> Path: + prepared_root = _prepared_root(root) + artifact = HypothesisArtifact.from_hypothesis(hypothesis) + content = (artifact.model_dump_json(indent=2) + "\n").encode("utf-8") + try: + with open_directory_chain( + prepared_root, + (".workspace",), + create=True, + ) as workspace: + _publish_workspace_ignore(workspace) + with open_child_directory(workspace, "hypotheses", create=True) as directory: + publish_idempotent( + directory, + f"{hypothesis.id.value}.json", + content, + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=hypothesis.id.value, + ) + except SafeFileFailure as error: + raise _storage_failure(error, hypothesis.id.value) from None + except OSError: + raise HypothesisFailure( + HypothesisErrorCode.WRITE_FAILED, + hypothesis.id.value, + ) from None + return Path(".workspace/hypotheses") / f"{hypothesis.id.value}.json" + + +def _prepared_root(root: Path) -> Path: + try: + resolved = root.resolve(strict=True) + except (OSError, RuntimeError): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, "workspace_root") from None + if not all(_regular_marker(resolved / name) for name in _WORKSPACE_MARKERS): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, "workspace_root") + return resolved + + +def _regular_marker(path: Path) -> bool: + try: + return stat.S_ISREG(path.lstat().st_mode) and not path.is_symlink() + except OSError: + return False + + +def _require_regular_target(root: Path, relative: Path) -> None: + path = root / relative + try: + metadata = path.lstat() + path.resolve(strict=True).relative_to(root) + except (OSError, ValueError): + raise HypothesisFailure(HypothesisErrorCode.INVALID_TARGET, relative.as_posix()) from None + if not stat.S_ISREG(metadata.st_mode) or path.is_symlink(): + raise HypothesisFailure(HypothesisErrorCode.INVALID_TARGET, relative.as_posix()) + + +def _publish_workspace_ignore(workspace: int) -> None: + publish_idempotent( + workspace, + ".gitignore", + b"*\n", + maximum_bytes=16, + subject=".workspace", + ) + + +def _git(root: Path, *arguments: str) -> str: + result = subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, arguments[0]) + return result.stdout.strip() + + +def _storage_failure(error: SafeFileFailure, hypothesis_id: str) -> HypothesisFailure: + if error.code is SafeFileErrorCode.CONFLICT: + code = HypothesisErrorCode.HYPOTHESIS_CONFLICT + elif error.code is SafeFileErrorCode.TOO_LARGE: + code = HypothesisErrorCode.HYPOTHESIS_TOO_LARGE + else: + code = HypothesisErrorCode.WRITE_FAILED + return HypothesisFailure(code, hypothesis_id) diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index cc50329..9bdb0fc 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -4,7 +4,8 @@ from __future__ import annotations import os -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from datetime import datetime from enum import StrEnum from importlib.resources import files @@ -44,6 +45,17 @@ VerifierResult, VerifierVerdict, ) +from ofw.evolution import ( + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateExecutionService, + CandidateGitGateway, + FileHypothesisRepository, + HypothesisObservation, + HypothesisService, + LangfuseCandidateTraceLocator, + RecordHypothesisInput, +) from ofw.observability.langfuse.contracts import LangfuseProject from ofw.observability.langfuse.domain import TraceId from ofw.observability.langfuse.trace_query import ( @@ -64,7 +76,7 @@ WorkspacePreparationObservation, WorkspacePreparationService, ) -from ofw.preparation.harbor import HarborBaselineRunner +from ofw.preparation.harbor import HarborBaselineRunner, HarborExperimentRunner from ofw.preparation.worktree import GitWorktreeGateway QueryInput = TypeVar("QueryInput") @@ -86,8 +98,8 @@ instructions=( "Prepare isolated ITSM harness workspaces, read bounded Langfuse trace evidence, and " "record authoritative outcomes, compact failure diagnoses, exact patterns, and " - "evidence-bound curations. Never infer outcomes, mutate traces, or copy trace payloads " - "into local storage." + "evidence-backed hypotheses and isolated candidates. Never infer outcomes, mutate " + "traces, copy trace payloads into local storage, or broaden candidate edit authority." ), log_level="DEBUG", ) @@ -156,6 +168,33 @@ def _curation_service() -> FailureCurationService: return FailureCurationService(FileFailureCurationWorkspace()) +def _hypothesis_service() -> HypothesisService: + workspace = FileFailureWorkspace() + return HypothesisService( + pattern_miner=FailurePatternMiningService(workspace), + repository=FileHypothesisRepository(), + ) + + +@contextmanager +def _candidate_service() -> Iterator[CandidateExecutionService]: + client = _client() + try: + store = _outcome_store() + try: + yield CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=HarborExperimentRunner(), + trace_locator=LangfuseCandidateTraceLocator(client), + outcome_store=store, + ) + finally: + store.close() + finally: + client.close() + + def _program_template(name: str) -> str: content = files("ofw.preparation.templates").joinpath(name).read_bytes() if len(content) > _PROGRAM_TEMPLATE_LIMIT_BYTES: @@ -297,6 +336,21 @@ def record_failure_curation( return _curation_service().record(request) +@server.tool(annotations=record_write, structured_output=True) +def record_hypothesis(request: RecordHypothesisInput) -> HypothesisObservation: + """Record one exact evidence-backed hypothesis, then stop before candidate editing.""" + return _hypothesis_service().record(request) + + +@server.tool(annotations=record_write, structured_output=True) +def execute_candidate( + request: CandidateExecutionInput, +) -> CandidateExecutionObservation: + """Create, seal, launch, or poll one exact hypothesis candidate.""" + with _candidate_service() as service: + return service.execute(request) + + def main() -> None: """Run the OpenFlywheel MCP server over stdio.""" server.run(transport="stdio") diff --git a/src/ofw/preparation/__init__.py b/src/ofw/preparation/__init__.py index 832cb6f..96ce3d6 100644 --- a/src/ofw/preparation/__init__.py +++ b/src/ofw/preparation/__init__.py @@ -5,6 +5,10 @@ BaselineRun, BaselineRunner, BaselineSummary, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, PreparationErrorCode, PreparationFailure, PreparationPhase, @@ -14,6 +18,12 @@ WorkspaceGateway, WorkspacePreparationObservation, ) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) from ofw.preparation.service import WorkspacePreparationService __all__ = [ @@ -21,6 +31,14 @@ "BaselineRun", "BaselineRunner", "BaselineSummary", + "ExperimentControls", + "ExperimentRun", + "ExperimentSummary", + "ExperimentTrial", + "ExperimentPolicyErrorCode", + "ExperimentPolicyFailure", + "ExperimentPolicySnapshot", + "FileExperimentPolicyRepository", "PreparationErrorCode", "PreparationFailure", "PreparationPhase", diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 17eaddf..3804eae 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -3,15 +3,16 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime, timedelta from enum import StrEnum from pathlib import Path from typing import Annotated, Protocol from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, field_validator -_EXPERIMENT_PATTERN = r"[a-z0-9]+(?:-[a-z0-9]+)*" -_REF_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._/@-]*" -_COMMIT_PATTERN = r"[0-9a-f]{40}" +_EXPERIMENT_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" +_REF_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._/@-]*$" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" def _normalized_score(value: object) -> float: @@ -28,13 +29,21 @@ def _numeric_float(value: object, field: str) -> float: return float(value) +def _bounded_path_input(value: object) -> object: + if isinstance(value, (str, Path)) and ( + "\x00" in str(value) or len(str(value).encode("utf-8")) > 1024 + ): + raise ValueError("path input must be bounded text") + return value + + ExperimentIdentifier = Annotated[ str, Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN), ] GitReference = Annotated[str, Field(min_length=1, max_length=256, pattern=_REF_PATTERN)] GoalText = Annotated[str, Field(min_length=1, max_length=2000)] -PathValue = Annotated[Path, Field(strict=False)] +PathValue = Annotated[Path, BeforeValidator(_bounded_path_input), Field(strict=False)] TaskCount = Annotated[int, Field(strict=True, ge=1, le=500)] IterationCount = Annotated[int, Field(strict=True, ge=1, le=100)] DurationSeconds = Annotated[int, Field(strict=True, ge=60, le=172800)] @@ -74,6 +83,7 @@ class PrepareWorkspaceInput(StrictModel): quality_target: NormalizedScore max_iterations: IterationCount no_improvement_limit: IterationCount + reuse_existing_baseline: bool = False max_cost_per_task_usd: PositiveMetric | None = None max_latency_seconds: PositiveMetric | None = None max_baseline_seconds: DurationSeconds @@ -93,12 +103,12 @@ def validate_absolute_paths(cls, value: Path) -> Path: @field_validator("harbor_config") @classmethod def validate_harbor_config(cls, value: Path) -> Path: - return _relative_path(value, "harbor_config") + return contained_relative_path(value, "harbor_config") @field_validator("editable_paths") @classmethod def validate_editable_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: - normalized = tuple(_relative_path(value, "editable_paths") for value in values) + normalized = tuple(contained_relative_path(value, "editable_paths") for value in values) if len(set(normalized)) != len(normalized): raise ValueError("editable_paths must be unique") return normalized @@ -133,6 +143,9 @@ class PreparationErrorCode(StrEnum): INVALID_BASELINE_RESULT = "invalid_baseline_result" PREPARATION_BUSY = "preparation_busy" GIT_FAILED = "git_failed" + POLICY_CONFLICT = "policy_conflict" + POLICY_SNAPSHOT_REQUIRED = "policy_snapshot_required" + POLICY_WRITE_FAILED = "policy_write_failed" class WorkspacePreparationObservation(StrictModel): @@ -164,7 +177,14 @@ class WorkspacePreparationObservation(StrictModel): @dataclass(frozen=True, slots=True) class BaselineConfiguration: model: str - task_count: int + task_ids: tuple[str, ...] + benchmark_config_digest: str + verifier: str + environment: str + + @property + def task_count(self) -> int: + return len(self.task_ids) @dataclass(frozen=True, slots=True) @@ -177,6 +197,7 @@ class BaselineRun: log_path: Path worktree_path: Path initialization_commit: str + controls: ExperimentControls @dataclass(frozen=True, slots=True) @@ -188,6 +209,60 @@ class BaselineSummary: unsupported_reward_trials: int +@dataclass(frozen=True, slots=True) +class ExperimentControls: + model: str + task_ids: tuple[str, ...] + benchmark_config_digest: str + verifier: str + environment: str + concurrency: int + max_retries: int + + +@dataclass(frozen=True, slots=True) +class ExperimentRun: + run_id: str + benchmark_root: Path + harbor_executable: Path + harbor_config: Path + job_path: Path + log_path: Path + source_root: Path + release: str + session_id: str + controls: ExperimentControls + + +@dataclass(frozen=True, slots=True) +class ExperimentTrial: + task_id: str + task_checksum: str + exception: bool + verdict: str | None + reward: float | None + started_at: datetime + finished_at: datetime + evaluated_at: datetime + evidence: tuple[str, ...] + + def __post_init__(self) -> None: + timestamps = (self.started_at, self.finished_at, self.evaluated_at) + if any(value.utcoffset() != timedelta(0) for value in timestamps): + raise ValueError("trial timestamps must be UTC") + if self.started_at >= self.finished_at or self.finished_at > self.evaluated_at: + raise ValueError("trial timestamps must be ordered") + + @property + def latency_seconds(self) -> float: + return (self.finished_at - self.started_at).total_seconds() + + +@dataclass(frozen=True, slots=True) +class ExperimentSummary: + trials: tuple[ExperimentTrial, ...] + + @dataclass(frozen=True, slots=True) class PreparedGitWorkspace: branch_name: str @@ -225,7 +300,18 @@ def __init__(self, code: PreparationErrorCode, subject: str) -> None: super().__init__(f"{code.value}: {subject}") -def _relative_path(value: Path, field: str) -> Path: +def contained_relative_path(value: Path, field: str) -> Path: + _require_relative_shape(value, field) + _require_bounded_path_text(value, field) + return value + + +def _require_relative_shape(value: Path, field: str) -> None: if value.is_absolute() or ".." in value.parts or value == Path("."): raise ValueError(f"{field} must be a contained relative path") - return value + + +def _require_bounded_path_text(value: Path, field: str) -> None: + text = value.as_posix() + if "\x00" in text or len(text.encode("utf-8")) > 1024: + raise ValueError(f"{field} must be a bounded text path") diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 416f425..383cca3 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -2,9 +2,12 @@ from __future__ import annotations +import hashlib import os +import signal import subprocess # nosec B404 from dataclasses import dataclass +from datetime import datetime from pathlib import Path from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError @@ -13,6 +16,10 @@ BaselineConfiguration, BaselineRun, BaselineSummary, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, PreparationErrorCode, PreparationFailure, PrepareWorkspaceInput, @@ -30,12 +37,16 @@ class _WireModel(BaseModel): class _HarborAgentWire(_WireModel): - name: str - model_name: str + name: str = Field(min_length=1, max_length=256) + model_name: str = Field(min_length=1, max_length=256) class _HarborTaskWire(_WireModel): - path: str + path: str = Field(min_length=1, max_length=256) + + +class _HarborTaskIdWire(_WireModel): + path: str = Field(min_length=1, max_length=256) class _HarborConfigWire(_WireModel): @@ -57,8 +68,21 @@ class _HarborVerifierResultWire(_WireModel): verdict: str | None = None +class _HarborExecutionWire(_WireModel): + started_at: datetime + finished_at: datetime + + +class _HarborVerifierWire(_WireModel): + finished_at: datetime + + class _HarborTrialResultWire(_WireModel): + task_id: str | _HarborTaskIdWire | None = None + task_checksum: str | None = None exception_info: JsonValue | None = None + agent_execution: _HarborExecutionWire | None = None + verifier: _HarborVerifierWire | None = None verifier_result: _HarborVerifierResultWire | None = None @@ -71,44 +95,64 @@ class _Credentials: langfuse_base_url: str -class HarborBaselineRunner: - """Launch one sequential ITSM Harbor job and parse its bounded results.""" - - def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: - _executable(request.harbor_executable) - config_path = _contained(request.benchmark_root, request.harbor_config) - config = _parse_config(config_path) - if len(config.tasks) != request.expected_task_count: - raise PreparationFailure( - PreparationErrorCode.TASK_COUNT_MISMATCH, - str(len(config.tasks)), - ) +class HarborExperimentRunner: + """Validate, launch, and normalize one deterministic Harbor experiment.""" + + def validate( + self, + benchmark_root: Path, + harbor_executable: Path, + harbor_config: Path, + *, + require_credentials: bool = True, + ) -> ExperimentControls: + _executable(harbor_executable) + config_path = _contained(benchmark_root, harbor_config) + config, config_content = _parse_config(config_path) agent = config.agents[0] if agent.name != _AGENT_NAME: raise PreparationFailure( PreparationErrorCode.INVALID_HARBOR_CONFIG, "agent", ) - _validate_source_adapter(request.benchmark_root) - _credentials() - return BaselineConfiguration(model=agent.model_name, task_count=len(config.tasks)) + task_ids = tuple(task.path for task in config.tasks) + if len(set(task_ids)) != len(task_ids): + raise PreparationFailure( + PreparationErrorCode.INVALID_HARBOR_CONFIG, + "tasks", + ) + _validate_source_adapter(benchmark_root) + if require_credentials: + _credentials() + return ExperimentControls( + model=agent.model_name, + task_ids=task_ids, + benchmark_config_digest=( + f"sha256:{hashlib.sha256(config_content.encode('utf-8')).hexdigest()}" + ), + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ) - def start(self, run: BaselineRun) -> int: + def start(self, run: ExperimentRun) -> int: if run.job_path.exists(): raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "job_path") + _require_run_controls(self, run) command = ( str(_executable(run.harbor_executable)), "run", "--config", str(run.harbor_config), "--job-name", - run.experiment_id, + run.run_id, "--jobs-dir", str(run.job_path.parent), "--n-concurrent", - "1", + str(run.controls.concurrency), "--max-retries", - "0", + str(run.controls.max_retries), "--yes", ) run.log_path.parent.mkdir(parents=True, exist_ok=True) @@ -130,6 +174,78 @@ def start(self, run: BaselineRun) -> int: ) from error return process.pid + def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: + root = _finished_job_result(run.job_path) + if root is None: + return None + trials = _experiment_trials(run) + _validate_experiment_trials(root.n_total_trials, trials, run.controls.task_ids) + return ExperimentSummary(trials) + + def cancel(self, run: ExperimentRun, process_id: int | None) -> None: + del run + if process_id is None: + return + try: + os.killpg(process_id, signal.SIGTERM) + except ProcessLookupError: + return + except OSError as error: + raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "cancel") from error + + +def _validate_experiment_trials( + expected_count: int, + trials: tuple[ExperimentTrial, ...], + expected_task_ids: tuple[str, ...], +) -> None: + if len(trials) > expected_count: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "trial count", + ) + if len(trials) != expected_count: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "terminal trial count", + ) + if tuple(trial.task_id for trial in trials) != expected_task_ids: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "task ids", + ) + + +class HarborBaselineRunner: + """Compatibility adapter preserving baseline preparation behavior.""" + + def __init__(self) -> None: + self._runner = HarborExperimentRunner() + + def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: + controls = self._runner.validate( + request.benchmark_root, + request.harbor_executable, + request.harbor_config, + require_credentials=False, + ) + if len(controls.task_ids) != request.expected_task_count: + raise PreparationFailure( + PreparationErrorCode.TASK_COUNT_MISMATCH, + str(len(controls.task_ids)), + ) + _credentials() + return BaselineConfiguration( + model=controls.model, + task_ids=controls.task_ids, + benchmark_config_digest=controls.benchmark_config_digest, + verifier=controls.verifier, + environment=controls.environment, + ) + + def start(self, run: BaselineRun) -> int: + return self._runner.start(_baseline_experiment_run(run, run.controls)) + def summarize(self, run: BaselineRun) -> BaselineSummary | None: root = _finished_job_result(run.job_path) if root is None: @@ -191,10 +307,11 @@ def _contained(root: Path, relative: Path) -> Path: return resolved -def _parse_config(path: Path) -> _HarborConfigWire: +def _parse_config(path: Path) -> tuple[_HarborConfigWire, str]: try: - return _HarborConfigWire.model_validate_json(_bounded_text(path, _MAX_CONFIG_BYTES)) - except (OSError, ValidationError, ValueError) as error: + content = _bounded_text(path, _MAX_CONFIG_BYTES) + return _HarborConfigWire.model_validate_json(content), content + except (OSError, UnicodeError, ValidationError, ValueError) as error: raise PreparationFailure( PreparationErrorCode.INVALID_HARBOR_CONFIG, path.name, @@ -254,6 +371,103 @@ def _trial_results(job_path: Path) -> tuple[_HarborTrialResultWire, ...]: return tuple(_parse_trial_result(path) for path in paths if path.exists()) +def _experiment_trials(run: ExperimentRun) -> tuple[ExperimentTrial, ...]: + trials: list[ExperimentTrial] = [] + for directory in sorted( + (child for child in run.job_path.iterdir() if child.is_dir()), + key=_path_name, + ): + result_path = directory / "result.json" + if result_path.exists(): + trials.append(_experiment_trial(run, directory.name, _parse_trial_result(result_path))) + return _ordered_trials(tuple(trials), run.controls.task_ids) + + +def _path_name(path: Path) -> str: + return path.name + + +def _ordered_trials( + trials: tuple[ExperimentTrial, ...], + task_ids: tuple[str, ...], +) -> tuple[ExperimentTrial, ...]: + ordered: list[ExperimentTrial] = [] + for task_id in task_ids: + matching = _matching_trials(trials, task_id) + if len(matching) != 1: + raise PreparationFailure(PreparationErrorCode.INVALID_BASELINE_RESULT, "task ids") + ordered.append(matching[0]) + if len(ordered) != len(trials): + raise PreparationFailure(PreparationErrorCode.INVALID_BASELINE_RESULT, "task ids") + return tuple(ordered) + + +def _matching_trials( + trials: tuple[ExperimentTrial, ...], + task_id: str, +) -> tuple[ExperimentTrial, ...]: + return tuple(trial for trial in trials if trial.task_id == task_id) + + +def _experiment_trial( + run: ExperimentRun, + directory_name: str, + wire: _HarborTrialResultWire, +) -> ExperimentTrial: + task_name, task_checksum, execution, verifier_wire = _required_trial_fields( + wire, + directory_name, + ) + verifier = wire.verifier_result + reward = None if verifier is None or verifier.rewards is None else verifier.rewards.reward + verdict = None if verifier is None else verifier.verdict + try: + return ExperimentTrial( + task_id=task_name, + task_checksum=task_checksum, + exception=wire.exception_info is not None, + verdict=verdict, + reward=reward, + started_at=execution.started_at, + finished_at=execution.finished_at, + evaluated_at=verifier_wire.finished_at, + evidence=( + f"harbor://{run.run_id}/{directory_name}/result.json", + f"harbor://{run.run_id}/{directory_name}/verifier", + ), + ) + except ValueError: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "trial timestamps", + ) from None + + +def _required_trial_fields( + wire: _HarborTrialResultWire, + directory_name: str, +) -> tuple[str, str, _HarborExecutionWire, _HarborVerifierWire]: + task_id = wire.task_id + if task_id is None: + raise _invalid_trial(directory_name) + if isinstance(task_id, _HarborTaskIdWire): + task_id = task_id.path + if wire.task_checksum is None: + raise _invalid_trial(directory_name) + if wire.agent_execution is None: + raise _invalid_trial(directory_name) + if wire.verifier is None: + raise _invalid_trial(directory_name) + return task_id, wire.task_checksum, wire.agent_execution, wire.verifier + + +def _invalid_trial(directory_name: str) -> PreparationFailure: + return PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + directory_name, + ) + + def _result_sort_key(path: Path) -> str: return path.parent.name @@ -307,7 +521,7 @@ def _required_any(primary: str, fallback: str) -> str: return value.strip() -def _process_environment(run: BaselineRun) -> dict[str, str]: +def _process_environment(run: ExperimentRun) -> dict[str, str]: credentials = _credentials() environment = dict(os.environ) environment.update( @@ -317,16 +531,52 @@ def _process_environment(run: BaselineRun) -> dict[str, str]: "HERMES_LANGFUSE_PUBLIC_KEY": credentials.langfuse_public_key, "HERMES_LANGFUSE_SECRET_KEY": credentials.langfuse_secret_key, "HERMES_LANGFUSE_BASE_URL": credentials.langfuse_base_url, - "HERMES_LANGFUSE_ENV": "itsm-bench", - "HERMES_LANGFUSE_RELEASE": run.initialization_commit, - "HERMES_LANGFUSE_SESSION_ID": run.experiment_id, - _SOURCE_ENVIRONMENT_NAME: str(run.worktree_path), + "HERMES_LANGFUSE_ENV": run.controls.environment, + "HERMES_LANGFUSE_RELEASE": run.release, + "HERMES_LANGFUSE_SESSION_ID": run.session_id, + _SOURCE_ENVIRONMENT_NAME: str(run.source_root), "PYTHONPATH": _python_path(run.benchmark_root, environment.get("PYTHONPATH")), } ) return environment +def _require_run_controls(runner: HarborExperimentRunner, run: ExperimentRun) -> None: + relative_config = _relative_run_config(run.benchmark_root, run.harbor_config) + actual = runner.validate( + run.benchmark_root, + run.harbor_executable, + relative_config, + ) + if actual != run.controls: + raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") + + +def _relative_run_config(benchmark_root: Path, harbor_config: Path) -> Path: + try: + return harbor_config.resolve(strict=True).relative_to(benchmark_root.resolve(strict=True)) + except (OSError, ValueError): + raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") from None + + +def _baseline_experiment_run( + run: BaselineRun, + controls: ExperimentControls, +) -> ExperimentRun: + return ExperimentRun( + run_id=run.experiment_id, + benchmark_root=run.benchmark_root, + harbor_executable=run.harbor_executable, + harbor_config=run.harbor_config, + job_path=run.job_path, + log_path=run.log_path, + source_root=run.worktree_path, + release=run.initialization_commit, + session_id=run.experiment_id, + controls=controls, + ) + + def _python_path(root: Path, existing: str | None) -> str: if existing is None or not existing: return str(root) diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py new file mode 100644 index 0000000..7f6929a --- /dev/null +++ b/src/ofw/preparation/policy.py @@ -0,0 +1,318 @@ +"""Canonical prepared-experiment policy stored outside the worktree.""" + +from __future__ import annotations + +import hashlib +import re +import subprocess # nosec B404 +from contextlib import AbstractContextManager +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import Field, ValidationError, field_validator, model_validator + +from ofw.preparation.contracts import ( + BaselineConfiguration, + EditablePaths, + ExperimentIdentifier, + GoalText, + IterationCount, + NormalizedScore, + PositiveMetric, + PreparedGitWorkspace, + PrepareWorkspaceInput, + StrictModel, + contained_relative_path, +) +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_directory_chain, + publish_idempotent, + read_bounded, +) + +_POLICY_LIMIT_BYTES = 256 * 1024 +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +_TASK_ID_PATTERN = r"^[^\x00]+$" +TaskIds = Annotated[ + tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], + Field(min_length=1, max_length=500), +] + + +class _ExperimentPolicyContent(StrictModel): + schema_version: Literal[1] = 1 + experiment_id: ExperimentIdentifier + branch_name: str = Field(min_length=1, max_length=256) + base_commit: str = Field(pattern=_COMMIT_PATTERN) + initialization_commit: str = Field(pattern=_COMMIT_PATTERN) + editable_paths: EditablePaths + goal: GoalText + quality_target: NormalizedScore + max_iterations: IterationCount + no_improvement_limit: IterationCount + baseline_reused: bool = False + max_cost_per_task_usd: PositiveMetric | None = None + max_latency_seconds: PositiveMetric | None = None + max_baseline_seconds: int = Field(strict=True, ge=60, le=172800) + benchmark: Literal["itsm-bench"] = "itsm-bench" + benchmark_config_digest: str = Field(pattern=_DIGEST_PATTERN) + task_ids: TaskIds + model: str = Field(min_length=1, max_length=256) + verifier: str = Field(min_length=1, max_length=128) + environment: str = Field(min_length=1, max_length=128) + concurrency: Literal[1] = 1 + max_retries: Literal[0] = 0 + + @field_validator("task_ids") + @classmethod + def validate_task_ids(cls, values: tuple[str, ...]) -> tuple[str, ...]: + if len(set(values)) != len(values): + raise ValueError("task_ids must be unique") + return values + + @field_validator("editable_paths") + @classmethod + def validate_editable_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: + normalized = tuple(contained_relative_path(path, "editable_paths") for path in values) + _require_unique_editable_paths(normalized) + return normalized + + +class ExperimentPolicySnapshot(_ExperimentPolicyContent): + """Immutable authority captured from validated preparation inputs and results.""" + + controls_digest: str = Field(pattern=_DIGEST_PATTERN) + + @model_validator(mode="after") + def validate_controls_digest(self) -> ExperimentPolicySnapshot: + if self.controls_digest != self.recomputed_controls_digest(): + raise ValueError("controls_digest does not match canonical policy") + return self + + def recomputed_controls_digest(self) -> str: + content = _content_from_snapshot(self) + return _digest(content.model_dump_json()) + + +class ExperimentPolicyErrorCode(StrEnum): + POLICY_SNAPSHOT_REQUIRED = "policy_snapshot_required" + POLICY_INVALID = "policy_invalid" + POLICY_CONFLICT = "policy_conflict" + POLICY_TOO_LARGE = "policy_too_large" + POLICY_WRITE_FAILED = "policy_write_failed" + + +class ExperimentPolicyFailure(Exception): + """Typed sanitized policy persistence failure.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: ExperimentPolicyErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +class FileExperimentPolicyRepository: + """Publish and reload one canonical policy in the Git common control directory.""" + + def publish(self, control_directory: Path, policy: ExperimentPolicySnapshot) -> Path: + content = (policy.model_dump_json(indent=2) + "\n").encode("utf-8") + try: + with _open_control_directory(control_directory) as directory: + publish_idempotent( + directory, + "policy.json", + content, + maximum_bytes=_POLICY_LIMIT_BYTES, + subject=policy.experiment_id, + ) + except SafeFileFailure as error: + raise _publication_failure(error, policy.experiment_id) from None + except OSError: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_WRITE_FAILED, + policy.experiment_id, + ) from None + return control_directory / "policy.json" + + def load(self, workspace_root: Path, experiment_id: str) -> ExperimentPolicySnapshot: + _require_experiment_id(experiment_id) + control_directory = _control_directory(workspace_root, experiment_id) + content = _read_policy_bytes(control_directory, experiment_id) + try: + policy = ExperimentPolicySnapshot.model_validate_json(content) + except (ValidationError, ValueError, UnicodeError): + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, + experiment_id, + ) from None + if policy.experiment_id != experiment_id: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, + experiment_id, + ) + return policy + + +def build_experiment_policy( + request: PrepareWorkspaceInput, + prepared: PreparedGitWorkspace, + baseline: BaselineConfiguration, +) -> ExperimentPolicySnapshot: + """Derive the authoritative snapshot solely from validated preparation values.""" + content = _ExperimentPolicyContent( + experiment_id=request.experiment_id, + branch_name=prepared.branch_name, + base_commit=prepared.base_commit, + initialization_commit=prepared.initialization_commit, + editable_paths=request.editable_paths, + goal=request.goal, + quality_target=request.quality_target, + max_iterations=request.max_iterations, + no_improvement_limit=request.no_improvement_limit, + baseline_reused=request.reuse_existing_baseline, + max_cost_per_task_usd=request.max_cost_per_task_usd, + max_latency_seconds=request.max_latency_seconds, + max_baseline_seconds=request.max_baseline_seconds, + benchmark_config_digest=baseline.benchmark_config_digest, + task_ids=baseline.task_ids, + model=baseline.model, + verifier=baseline.verifier, + environment=baseline.environment, + ) + return _snapshot_from_content(content) + + +def _content_from_snapshot(policy: ExperimentPolicySnapshot) -> _ExperimentPolicyContent: + return _ExperimentPolicyContent( + experiment_id=policy.experiment_id, + branch_name=policy.branch_name, + base_commit=policy.base_commit, + initialization_commit=policy.initialization_commit, + editable_paths=policy.editable_paths, + goal=policy.goal, + quality_target=policy.quality_target, + max_iterations=policy.max_iterations, + no_improvement_limit=policy.no_improvement_limit, + baseline_reused=policy.baseline_reused, + max_cost_per_task_usd=policy.max_cost_per_task_usd, + max_latency_seconds=policy.max_latency_seconds, + max_baseline_seconds=policy.max_baseline_seconds, + benchmark_config_digest=policy.benchmark_config_digest, + task_ids=policy.task_ids, + model=policy.model, + verifier=policy.verifier, + environment=policy.environment, + ) + + +def _snapshot_from_content(content: _ExperimentPolicyContent) -> ExperimentPolicySnapshot: + return ExperimentPolicySnapshot( + experiment_id=content.experiment_id, + branch_name=content.branch_name, + base_commit=content.base_commit, + initialization_commit=content.initialization_commit, + editable_paths=content.editable_paths, + goal=content.goal, + quality_target=content.quality_target, + max_iterations=content.max_iterations, + no_improvement_limit=content.no_improvement_limit, + baseline_reused=content.baseline_reused, + max_cost_per_task_usd=content.max_cost_per_task_usd, + max_latency_seconds=content.max_latency_seconds, + max_baseline_seconds=content.max_baseline_seconds, + benchmark_config_digest=content.benchmark_config_digest, + task_ids=content.task_ids, + model=content.model, + verifier=content.verifier, + environment=content.environment, + controls_digest=_digest(content.model_dump_json()), + ) + + +def _control_directory(workspace_root: Path, experiment_id: str) -> Path: + result = subprocess.run( + ("git", "-C", str(workspace_root), "rev-parse", "--git-common-dir"), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise ExperimentPolicyFailure(ExperimentPolicyErrorCode.POLICY_INVALID, experiment_id) + common = Path(result.stdout.strip()) + if not common.is_absolute(): + common = workspace_root / common + return common / "ofw" / "preparations" / experiment_id + + +def _open_control_directory(control_directory: Path) -> AbstractContextManager[int]: + common = control_directory.parents[2] + return open_directory_chain( + common, + ("ofw", "preparations", control_directory.name), + create=False, + ) + + +def _read_policy_bytes(control_directory: Path, experiment_id: str) -> bytes: + try: + with _open_control_directory(control_directory) as directory: + return read_bounded( + directory, + "policy.json", + maximum_bytes=_POLICY_LIMIT_BYTES, + subject=experiment_id, + ) + except FileNotFoundError: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED, + experiment_id, + ) from None + except SafeFileFailure as error: + raise _read_failure(error, experiment_id) from None + except OSError: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, + experiment_id, + ) from None + + +def _require_experiment_id(experiment_id: str) -> None: + if re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", experiment_id) is None: + raise ExperimentPolicyFailure(ExperimentPolicyErrorCode.POLICY_INVALID, "experiment_id") + + +def _publication_failure( + error: SafeFileFailure, + experiment_id: str, +) -> ExperimentPolicyFailure: + if error.code is SafeFileErrorCode.CONFLICT: + code = ExperimentPolicyErrorCode.POLICY_CONFLICT + elif error.code is SafeFileErrorCode.TOO_LARGE: + code = ExperimentPolicyErrorCode.POLICY_TOO_LARGE + else: + code = ExperimentPolicyErrorCode.POLICY_WRITE_FAILED + return ExperimentPolicyFailure(code, experiment_id) + + +def _read_failure(error: SafeFileFailure, experiment_id: str) -> ExperimentPolicyFailure: + code = ( + ExperimentPolicyErrorCode.POLICY_TOO_LARGE + if error.code is SafeFileErrorCode.TOO_LARGE + else ExperimentPolicyErrorCode.POLICY_INVALID + ) + return ExperimentPolicyFailure(code, experiment_id) + + +def _digest(value: str) -> str: + return f"sha256:{hashlib.sha256(value.encode('utf-8')).hexdigest()}" + + +def _require_unique_editable_paths(paths: tuple[Path, ...]) -> None: + if len(set(paths)) != len(paths): + raise ValueError("editable_paths must be unique") diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index 3705004..adef880 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -11,12 +11,14 @@ from datetime import UTC, datetime, timedelta from pathlib import Path -from pydantic import Field +from pydantic import BaseModel, ConfigDict, Field, ValidationError from ofw.preparation.contracts import ( + BaselineConfiguration, BaselineRun, BaselineRunner, BaselineSummary, + ExperimentControls, PreparationErrorCode, PreparationFailure, PreparationPhase, @@ -27,13 +29,20 @@ WorkspaceGateway, WorkspacePreparationObservation, ) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + FileExperimentPolicyRepository, + build_experiment_policy, +) -_COMMIT_PATTERN = r"[0-9a-f]{40}" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" class _PreparationStateWire(StrictModel): - schema_version: int = Field(default=1, ge=1, le=1) - request_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + schema_version: int = Field(default=2, ge=2, le=2) + request_digest: str = Field(pattern=_DIGEST_PATTERN) phase: PreparationPhase branch_name: str worktree_path: Path @@ -42,7 +51,13 @@ class _PreparationStateWire(StrictModel): program_path: Path job_path: Path log_path: Path - model: str + model: str = Field(min_length=1, max_length=256) + task_ids: tuple[str, ...] = Field(min_length=1, max_length=500) + benchmark_config_digest: str = Field(pattern=_DIGEST_PATTERN) + verifier: str = Field(min_length=1, max_length=128) + environment: str = Field(min_length=1, max_length=128) + baseline_reused: bool = False + policy_published: bool = False process_id: int | None = Field(default=None, strict=True, ge=1) started_at: datetime deadline_at: datetime @@ -54,6 +69,12 @@ class _PreparationStateWire(StrictModel): error_code: PreparationErrorCode | None = None +class _StateVersionWire(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True, strict=True) + + schema_version: int = Field(strict=True) + + @dataclass(frozen=True, slots=True) class _TerminalCounts: terminal_trials: int | None @@ -112,26 +133,50 @@ def _start( self._program, configuration, ) - job_path = request.benchmark_root / "jobs" / request.experiment_id - log_path = state_directory / "baseline.log" - run = _baseline_run(request, prepared, job_path, log_path) - started_at = datetime.now(UTC) - state = _PreparationStateWire( - request_digest=digest, - phase=PreparationPhase.RUNNING, - branch_name=prepared.branch_name, - worktree_path=prepared.worktree_path, - base_commit=prepared.base_commit, - initialization_commit=prepared.initialization_commit, - program_path=prepared.program_path, - job_path=job_path, - log_path=log_path, - model=configuration.model, - process_id=None, - started_at=started_at, - deadline_at=started_at + timedelta(seconds=request.max_baseline_seconds), - ) + state = _initial_state(request, state_directory, digest, prepared, configuration) _write_state(state_directory, state) + if request.reuse_existing_baseline: + return self._adopt_existing(request, state_directory, state) + return self._resume_start(request, state_directory, state) + + def _adopt_existing( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation: + summary = self._runner.summarize(_run_from_state(request, state)) + if summary is None: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "existing baseline", + ) + _publish_policy( + state_directory, + request, + _prepared_from_state(state), + _configuration_from_state(state), + ) + ready = _ready_state(_state_with_policy(state), summary, request.expected_task_count) + _write_state(state_directory, ready) + return _ready_observation(request, ready) + + def _resume_start( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation: + if not state.policy_published: + _publish_policy( + state_directory, + request, + _prepared_from_state(state), + _configuration_from_state(state), + ) + state = _state_with_policy(state) + _write_state(state_directory, state) + run = _run_from_state(request, state) try: process_id = self._runner.start(run) except PreparationFailure as error: @@ -157,9 +202,19 @@ def _poll( PreparationErrorCode.REQUEST_CONFLICT, request.experiment_id, ) - terminal = _terminal_observation(request, state) - if terminal is not None: - return terminal + if state.baseline_reused: + return self._adopt_existing(request, state_directory, state) + continuation = self._continue_existing(request, state_directory, state) + if continuation is not None: + return continuation + return self._poll_running_baseline(request, state_directory, state) + + def _poll_running_baseline( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation: run = _run_from_state(request, state) summary = self._runner.summarize(run) if summary is not None: @@ -176,6 +231,19 @@ def _poll( _write_state(state_directory, failed) return _persisted_failure_observation(request, failed) + def _continue_existing( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation | None: + terminal = _terminal_observation(request, state) + if terminal is not None: + return terminal + if state.process_id is None: + return self._resume_start(request, state_directory, state) + return None + def _directory(path: Path, field: str) -> Path: try: @@ -187,6 +255,60 @@ def _directory(path: Path, field: str) -> Path: return resolved +def _publish_policy( + state_directory: Path, + request: PrepareWorkspaceInput, + prepared: PreparedGitWorkspace, + configuration: BaselineConfiguration, +) -> None: + try: + FileExperimentPolicyRepository().publish( + state_directory, + build_experiment_policy(request, prepared, configuration), + ) + except ExperimentPolicyFailure as error: + code = ( + PreparationErrorCode.POLICY_CONFLICT + if error.code is ExperimentPolicyErrorCode.POLICY_CONFLICT + else PreparationErrorCode.POLICY_WRITE_FAILED + ) + raise PreparationFailure(code, request.experiment_id) from None + except (ValidationError, ValueError): + raise PreparationFailure( + PreparationErrorCode.POLICY_WRITE_FAILED, + request.experiment_id, + ) from None + + +def _initial_state( + request: PrepareWorkspaceInput, + state_directory: Path, + digest: str, + prepared: PreparedGitWorkspace, + configuration: BaselineConfiguration, +) -> _PreparationStateWire: + started_at = datetime.now(UTC) + return _PreparationStateWire( + request_digest=digest, + phase=PreparationPhase.RUNNING, + branch_name=prepared.branch_name, + worktree_path=prepared.worktree_path, + base_commit=prepared.base_commit, + initialization_commit=prepared.initialization_commit, + program_path=prepared.program_path, + job_path=request.benchmark_root / "jobs" / request.experiment_id, + log_path=state_directory / "baseline.log", + model=configuration.model, + task_ids=configuration.task_ids, + benchmark_config_digest=configuration.benchmark_config_digest, + verifier=configuration.verifier, + environment=configuration.environment, + baseline_reused=request.reuse_existing_baseline, + started_at=started_at, + deadline_at=started_at + timedelta(seconds=request.max_baseline_seconds), + ) + + def _compose_program(base: str, itsm: str) -> str: if not base.strip() or not itsm.strip(): raise ValueError("program templates must not be empty") @@ -218,12 +340,35 @@ def _read_state(state_directory: Path) -> _PreparationStateWire | None: if not path.exists(): return None try: - return _PreparationStateWire.model_validate_json(path.read_text(encoding="utf-8")) + content = path.read_text(encoding="utf-8") except (OSError, ValueError) as error: raise PreparationFailure( PreparationErrorCode.INVALID_BASELINE_RESULT, "state.json", ) from error + return _parse_state(content) + + +def _parse_state(content: str) -> _PreparationStateWire: + try: + version = _StateVersionWire.model_validate_json(content) + except (ValidationError, ValueError) as error: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "state.json", + ) from error + if version.schema_version != 2: + raise PreparationFailure( + PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED, + "state.json", + ) + try: + return _PreparationStateWire.model_validate_json(content) + except (ValidationError, ValueError) as error: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "state.json", + ) from error def _write_state(state_directory: Path, state: _PreparationStateWire) -> None: @@ -239,24 +384,6 @@ def _write_state(state_directory: Path, state: _PreparationStateWire) -> None: os.replace(temporary_path, path) -def _baseline_run( - request: PrepareWorkspaceInput, - prepared: PreparedGitWorkspace, - job_path: Path, - log_path: Path, -) -> BaselineRun: - return BaselineRun( - experiment_id=request.experiment_id, - benchmark_root=request.benchmark_root, - harbor_executable=request.harbor_executable, - harbor_config=request.benchmark_root / request.harbor_config, - job_path=job_path, - log_path=log_path, - worktree_path=prepared.worktree_path, - initialization_commit=prepared.initialization_commit, - ) - - def _run_from_state( request: PrepareWorkspaceInput, state: _PreparationStateWire, @@ -270,6 +397,35 @@ def _run_from_state( log_path=state.log_path, worktree_path=state.worktree_path, initialization_commit=state.initialization_commit, + controls=ExperimentControls( + model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + concurrency=1, + max_retries=0, + ), + ) + + +def _prepared_from_state(state: _PreparationStateWire) -> PreparedGitWorkspace: + return PreparedGitWorkspace( + branch_name=state.branch_name, + worktree_path=state.worktree_path, + base_commit=state.base_commit, + initialization_commit=state.initialization_commit, + program_path=state.program_path, + ) + + +def _configuration_from_state(state: _PreparationStateWire) -> BaselineConfiguration: + return BaselineConfiguration( + model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, ) @@ -309,6 +465,12 @@ def _terminal_state( job_path=state.job_path, log_path=state.log_path, model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + baseline_reused=state.baseline_reused, + policy_published=state.policy_published, process_id=state.process_id, started_at=state.started_at, deadline_at=state.deadline_at, @@ -348,12 +510,42 @@ def _state_with_process( job_path=state.job_path, log_path=state.log_path, model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + baseline_reused=state.baseline_reused, + policy_published=state.policy_published, process_id=process_id, started_at=state.started_at, deadline_at=state.deadline_at, ) +def _state_with_policy(state: _PreparationStateWire) -> _PreparationStateWire: + return _PreparationStateWire( + request_digest=state.request_digest, + phase=state.phase, + branch_name=state.branch_name, + worktree_path=state.worktree_path, + base_commit=state.base_commit, + initialization_commit=state.initialization_commit, + program_path=state.program_path, + job_path=state.job_path, + log_path=state.log_path, + model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + baseline_reused=state.baseline_reused, + policy_published=True, + process_id=state.process_id, + started_at=state.started_at, + deadline_at=state.deadline_at, + ) + + def _running_observation( request: PrepareWorkspaceInput, state: _PreparationStateWire, @@ -458,6 +650,17 @@ def _failure_observation( def _root_cause(code: PreparationErrorCode) -> str: + if code is PreparationErrorCode.POLICY_CONFLICT: + return "The experiment already has different immutable policy bytes." + if code is PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED: + return "The existing preparation predates the canonical policy snapshot." + specific = _specific_root_cause(code) + if specific is not None: + return specific + return "A validated workspace, Git, Harbor, or result boundary rejected the request." + + +def _specific_root_cause(code: PreparationErrorCode) -> str | None: if code is PreparationErrorCode.REQUEST_CONFLICT: return "The preparation ID is already bound to different canonical inputs." if code is PreparationErrorCode.TASK_COUNT_MISMATCH: @@ -466,7 +669,7 @@ def _root_cause(code: PreparationErrorCode) -> str: return "One or more required credential environment variables are absent." if code is PreparationErrorCode.BASELINE_TIMEOUT: return "The Harbor job did not publish a terminal result before its deadline." - return "A validated workspace, Git, Harbor, or result boundary rejected the request." + return None def _retry(code: PreparationErrorCode) -> str: @@ -474,6 +677,10 @@ def _retry(code: PreparationErrorCode) -> str: return "Poll the identical request after 30 seconds." if code is PreparationErrorCode.BASELINE_TIMEOUT: return "Inspect the bounded baseline log, then use a new experiment ID if rerunning." + if code is PreparationErrorCode.POLICY_CONFLICT: + return "Do not retry this experiment ID with different policy content." + if code is PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED: + return "Use a new experiment ID and prepare again; do not infer policy from YAML." return "Correct the reported configuration boundary, then retry without forcing Git state." @@ -482,6 +689,7 @@ def _stop_when(code: PreparationErrorCode) -> str: PreparationErrorCode.BRANCH_EXISTS, PreparationErrorCode.WORKTREE_EXISTS, PreparationErrorCode.REQUEST_CONFLICT, + PreparationErrorCode.POLICY_CONFLICT, ): - return "Stop until the existing experiment ownership is resolved explicitly." + return "Stop and use a new experiment ID unless the existing ownership is accepted." return "Stop if correction requires deleting, resetting, or overwriting user-owned data." diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index 288034d..c440f02 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -4,16 +4,19 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Improve the connected agent harness until the goal in `experiment_config.yaml` is met, -while respecting its quality, cost, latency, budget, and stopping constraints. +Record one evidence-backed hypothesis, execute its isolated candidate under the canonical +experiment policy, then stop before admission. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged -baseline. +baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an +existing terminal Harbor job was adopted). ## Authority -- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal, - benchmark, verifier, budget, and stopping conditions. +- The canonical experiment policy stored in the Git common control directory defines the + prepared branch and commits, exact editable surface, frozen controls, goal, benchmark, + verifier, budget, and stopping conditions. `experiment_config.yaml` is a human-readable copy, + not an internal authority interface. - The external verifier is authoritative for task outcome. The agent's completion claim is not proof of success. - Langfuse is the source of truth for trajectories, usage, cost, and latency. @@ -24,9 +27,9 @@ baseline. ## Editable and frozen surfaces -Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark, -held-out tasks, verifier, model, reasoning budget, observability identity, or this program -to improve a score. +Target only exact paths allowed by the canonical experiment policy. Edit them only in the +candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks, +verifier, model, reasoning budget, observability identity, or this program. Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control flow changes unless the evidence requires the combination. @@ -46,37 +49,22 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn expected improvement, and possible regressions. Stop if the evidence cannot distinguish between materially different changes. -### 4. Improve the harness +Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported +pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call +`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution. -Make the smallest change within the declared editable surface. Preserve frozen controls and -unrelated user changes. +### 4. Execute one candidate -### 5. Gate the change +Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling +candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated +worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the +returned candidate worktree, then call `execute_candidate` again with the identical request. -Run only the prepared experiment command and gates declared by the workspace. Compare -task-level verifier outcomes and report quality, cost, and latency separately. Missing or -errored trials remain visible and cannot disappear from the denominator. +Poll identical requests while the candidate is running. Retain its candidate ID, Git commit, +trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally, +change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree. -### 6. Keep or revert +## Package boundary -Keep the change only when the configured gate admits it. Otherwise revert only the current -iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken -the gate to admit a candidate. - -Commit each admitted improvement on the prepared `ofw/` branch before the -next iteration. Keep one hypothesis per commit and include `OFW-Experiment` and `OFW-Run` -trailers. Do not commit failed candidates, generated run artifacts, credentials, or changes -outside the editable surface. Do not push or open a pull request without explicit user -authorization. - -### 7. Repeat - -Return to step 2 with the newly recorded run. Stop when the configured goal is met, the -budget or iteration limit is exhausted, the no-improvement condition is reached, or required -authoritative evidence is unavailable. - -## Final report - -Report the accepted harness revision, verifier-backed quality, cost, latency, remaining -unverified trials, iteration count, and exact stopping reason. Do not claim improvement from -an agent-authored summary alone. +Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission: +do not gate, accept, merge, publish, push, or install the candidate. diff --git a/src/ofw/preparation/worktree.py b/src/ofw/preparation/worktree.py index 25cf4b6..85e0d78 100644 --- a/src/ofw/preparation/worktree.py +++ b/src/ofw/preparation/worktree.py @@ -191,6 +191,7 @@ def _render_experiment_config( f" harbor_config: {_yaml(request.harbor_config.as_posix())}\n" f" job_name: {_yaml(request.experiment_id)}\n" f" expected_task_count: {baseline.task_count}\n" + f" reuse_existing_baseline: {str(request.reuse_existing_baseline).lower()}\n" "harness:\n" f" branch: {_yaml(branch_name)}\n" f" base_commit: {_yaml(base_commit)}\n" diff --git a/src/ofw/safe_file.py b/src/ofw/safe_file.py new file mode 100644 index 0000000..a466c62 --- /dev/null +++ b/src/ofw/safe_file.py @@ -0,0 +1,245 @@ +"""Small descriptor-anchored primitives for bounded immutable local files.""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Iterator +from contextlib import contextmanager +from enum import StrEnum +from pathlib import Path +from uuid import uuid4 + +_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW +_CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW +_READ_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + + +class SafeFileErrorCode(StrEnum): + CONFLICT = "conflict" + DIRECTORY_CHANGED = "directory_changed" + INVALID_FILE = "invalid_file" + TOO_LARGE = "too_large" + + +class SafeFileFailure(Exception): + """Sanitized failure from a local immutable-file boundary.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: SafeFileErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +@contextmanager +def open_directory(path: Path) -> Iterator[int]: + """Open one directory without following links and recheck its identity on exit.""" + with open_directory_chain(path, (), create=False) as descriptor: + yield descriptor + + +@contextmanager +def open_child_directory(parent: int, name: str, *, create: bool) -> Iterator[int]: + """Open one child directory and reject replacement while its descriptor is in use.""" + descriptor = _open_child(parent, name, create=create) + try: + _require_child_identity(parent, name, descriptor) + yield descriptor + _require_child_identity(parent, name, descriptor) + finally: + os.close(descriptor) + + +@contextmanager +def open_directory_chain( + root: Path, + parts: tuple[str, ...], + *, + create: bool, +) -> Iterator[int]: + """Open a contained child chain and recheck every link before and after use.""" + anchor, existing_parts = _absolute_directory_parts(root) + all_parts = existing_parts + parts + anchor_descriptor = os.open(anchor, _DIRECTORY_FLAGS) + descriptors = [anchor_descriptor] + expected_anchor = _descriptor_identity(anchor_descriptor) + try: + for index, part in enumerate(all_parts): + child_create = create and index >= len(existing_parts) + descriptors.append(_open_child(descriptors[-1], part, create=child_create)) + _require_chain(anchor, all_parts, descriptors, expected_anchor) + yield descriptors[-1] + _require_chain(anchor, all_parts, descriptors, expected_anchor) + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def publish_idempotent( + directory: int, + name: str, + content: bytes, + *, + maximum_bytes: int, + subject: str, +) -> None: + """Publish new bytes once, accepting only an identical existing regular file.""" + _require_leaf(name) + _require_size(content, maximum_bytes, subject) + try: + _publish_new_file(directory, name, content) + except FileExistsError: + if read_bounded(directory, name, maximum_bytes=maximum_bytes, subject=subject) != content: + raise SafeFileFailure(SafeFileErrorCode.CONFLICT, subject) from None + + +def read_bounded( + directory: int, + name: str, + *, + maximum_bytes: int, + subject: str, +) -> bytes: + """Read one no-follow regular file through its containing directory descriptor.""" + _require_leaf(name) + try: + descriptor = os.open(name, _READ_FILE_FLAGS, dir_fd=directory) + except OSError as error: + if isinstance(error, FileNotFoundError): + raise + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, subject) from None + try: + _require_regular_file(descriptor, subject) + content = _read_descriptor(descriptor, maximum_bytes) + finally: + os.close(descriptor) + _require_size(content, maximum_bytes, subject) + return content + + +def _open_child(parent: int, name: str, *, create: bool) -> int: + _require_leaf(name) + if create: + _mkdir_if_missing(parent, name) + try: + return os.open(name, _DIRECTORY_FLAGS, dir_fd=parent) + except FileNotFoundError: + raise + except OSError: + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, name) from None + + +def _mkdir_if_missing(parent: int, name: str) -> None: + try: + os.mkdir(name, 0o700, dir_fd=parent) + except FileExistsError: + return + os.fsync(parent) + + +def _require_chain( + root: Path, + parts: tuple[str, ...], + descriptors: list[int], + expected_root: tuple[int, int], +) -> None: + _require_path_identity(root, expected_root) + for index, name in enumerate(parts): + _require_child_identity(descriptors[index], name, descriptors[index + 1]) + + +def _require_path_identity(path: Path, expected: tuple[int, int]) -> None: + try: + current = _stat_identity(os.stat(path, follow_symlinks=False)) + except OSError: + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, path.name) from None + if current != expected: + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, path.name) + + +def _require_child_identity(parent: int, name: str, descriptor: int) -> None: + try: + current = _stat_identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) + except OSError: + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, name) from None + if current != _descriptor_identity(descriptor): + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, name) + + +def _publish_new_file(directory: int, name: str, content: bytes) -> None: + temporary_name = f".ofw-{uuid4().hex}.tmp" + published = False + try: + write_new_file(directory, temporary_name, content) + os.link( + temporary_name, + name, + src_dir_fd=directory, + dst_dir_fd=directory, + follow_symlinks=False, + ) + published = True + finally: + _unlink_if_present(directory, temporary_name) + if published: + os.fsync(directory) + + +def write_new_file(directory: int, name: str, content: bytes) -> None: + """Create and fsync one exclusive no-follow file under a directory descriptor.""" + _require_leaf(name) + descriptor = os.open(name, _CREATE_FILE_FLAGS, 0o600, dir_fd=directory) + with os.fdopen(descriptor, "wb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + + +def _unlink_if_present(directory: int, name: str) -> None: + try: + os.unlink(name, dir_fd=directory) + except FileNotFoundError: + return + + +def _read_descriptor(descriptor: int, maximum_bytes: int) -> bytes: + chunks: list[bytes] = [] + remaining = maximum_bytes + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def _require_regular_file(descriptor: int, subject: str) -> None: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, subject) + + +def _require_size(content: bytes, maximum_bytes: int, subject: str) -> None: + if len(content) > maximum_bytes: + raise SafeFileFailure(SafeFileErrorCode.TOO_LARGE, subject) + + +def _require_leaf(name: str) -> None: + if not name or name in (".", "..") or "/" in name or "\x00" in name: + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, "file_name") + + +def _descriptor_identity(descriptor: int) -> tuple[int, int]: + return _stat_identity(os.fstat(descriptor)) + + +def _stat_identity(value: os.stat_result) -> tuple[int, int]: + return value.st_dev, value.st_ino + + +def _absolute_directory_parts(path: Path) -> tuple[Path, tuple[str, ...]]: + if not path.is_absolute(): + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, "directory") + return Path(path.anchor), path.parts[1:] diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py new file mode 100644 index 0000000..5a2fbca --- /dev/null +++ b/tests/test_candidate_execution.py @@ -0,0 +1,984 @@ +"""Candidate identity, Git isolation, execution, and authoritative receipts.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import ofw.evolution.candidate_service as candidate_service_module +from ofw.contracts import ComponentKind, Sha256Digest +from ofw.evaluation.langfuse import OutcomeScoreSubmission +from ofw.evaluation.outcome import OutcomeEvaluation +from ofw.evolution.candidate import ( + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateFailure, + CandidateId, + CandidatePhase, + CandidateStatus, + TraceMatch, + TraceMatchRequest, + candidate_policy_digest, +) +from ofw.evolution.candidate_git import CandidateGitGateway +from ofw.evolution.candidate_langfuse import LangfuseCandidateTraceLocator +from ofw.evolution.candidate_service import CandidateExecutionService +from ofw.evolution.hypothesis import ( + FailurePatternReference, + HarnessChangeTarget, + HarnessHypothesis, + HypothesisArtifact, + HypothesisId, +) +from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.observability.langfuse.domain import ( + JsonDocument, + ObservationId, + ObservationPage, + ObservationRecord, + ObservationType, + PageCursor, + ProjectId, + ScoreId, + TraceId, +) +from ofw.observability.langfuse.trace_query import ObservationRead +from ofw.preparation.contracts import ( + BaselineConfiguration, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, + PreparationErrorCode, + PreparationFailure, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import ( + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, + build_experiment_policy, +) + + +def _git(root: Path, *arguments: str, check: bool = True) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=check, + capture_output=True, + text=True, + ).stdout.strip() + + +def _authority(tmp_path: Path) -> tuple[Path, ExperimentPolicySnapshot, HarnessHypothesis]: + root = tmp_path / "accepted" + root.mkdir() + (root / "prompt.md").write_text("Original prompt.\n", encoding="utf-8") + (root / "tools.py").write_text("def tool() -> bool:\n return True\n", encoding="utf-8") + (root / ".gitignore").write_text(".env\n", encoding="utf-8") + (root / "PROGRAM.md").write_text("# Managed\n", encoding="utf-8") + (root / "experiment_config.yaml").write_text("managed: true\n", encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.name", "OpenFlywheel Test") + _git(root, "config", "user.email", "ofw@example.test") + _git(root, "add", ".") + _git(root, "commit", "-qm", "accepted experiment") + _git(root, "branch", "-m", "ofw/experiment-one") + commit = _git(root, "rev-parse", "HEAD") + benchmark = tmp_path / "benchmark" + benchmark.mkdir() + executable = tmp_path / "harbor" + executable.touch(mode=0o700) + request = PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=root, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=Path("config.json"), + expected_task_count=2, + editable_paths=(Path("prompt.md"), Path("tools.py")), + goal="Improve verifier-backed quality.", + quality_target=1.0, + max_iterations=3, + no_improvement_limit=2, + max_baseline_seconds=600, + ) + policy = build_experiment_policy( + request, + PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=root, + base_commit=commit, + initialization_commit=commit, + program_path=root / "PROGRAM.md", + ), + BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-1", "task-2"), + benchmark_config_digest="sha256:" + "a" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ), + ) + hypothesis = HarnessHypothesis( + id=HypothesisId("sha256:" + "b" * 64), + experiment_id=policy.experiment_id, + source_commit=commit, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), + patterns=( + FailurePatternReference( + "sha256:" + "c" * 64, + ("00000000-0000-0000-0000-000000000000",), + ), + ), + statement="Require a state check before finalizing.", + rationale="Supported failures share the same cause.", + target=HarnessChangeTarget(ComponentKind.PROMPT, (Path("prompt.md"),)), + expected_effect="The agent verifies completion.", + regression_risks=(), + ) + hypothesis = replace( + hypothesis, + id=HypothesisArtifact.from_hypothesis(hypothesis).recomputed_id(), + ) + control = root / ".git/ofw/preparations/experiment-one" + control.mkdir(parents=True) + FileExperimentPolicyRepository().publish(control, policy) + FileHypothesisRepository().store(root, hypothesis) + return root, policy, hypothesis + + +class _FakeRunner: + def __init__(self, controls: ExperimentControls) -> None: + self.controls = controls + self.runs: list[ExperimentRun] = [] + self.summary: ExperimentSummary | None = None + self.failure: PreparationFailure | None = None + self.start_failure: PreparationFailure | None = None + self.start_count = 0 + self.cancelled: list[tuple[ExperimentRun, int | None]] = [] + + def validate( + self, + benchmark_root: Path, + harbor_executable: Path, + harbor_config: Path, + ) -> ExperimentControls: + if self.failure is not None: + raise self.failure + return self.controls + + def start(self, run: ExperimentRun) -> int: + self.start_count += 1 + if self.start_failure is not None: + raise self.start_failure + self.runs.append(run) + return 123 + + def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: + return self.summary + + def cancel(self, run: ExperimentRun, process_id: int | None) -> None: + self.cancelled.append((run, process_id)) + + +class _FakeTraceLocator: + def __init__(self) -> None: + self.requests: list[TraceMatchRequest] = [] + self.blocker: CandidateBlockerCode | None = None + self.cost_usd: float | None = None + + def locate(self, request: TraceMatchRequest) -> TraceMatch: + self.requests.append(request) + if self.blocker is not None: + return TraceMatch(trace_id=None, blocker=self.blocker) + return TraceMatch( + trace_id=f"trace-{request.task_id}", + blocker=None, + cost_usd=self.cost_usd, + ) + + +class _FakeOutcomeStore: + def __init__(self) -> None: + self.outcomes: list[OutcomeEvaluation] = [] + self.failure: Exception | None = None + + def store(self, outcome: OutcomeEvaluation) -> OutcomeScoreSubmission: + if self.failure is not None: + raise self.failure + self.outcomes.append(outcome) + return OutcomeScoreSubmission( + ScoreId(f"score-{outcome.task_id.value}"), + outcome.trace_id, + ) + + +class _ObservationReader: + def __init__(self, page: ObservationPage) -> None: + self.page = page + self.queries: list[ObservationRead] = [] + + def read_observations(self, query: ObservationRead) -> ObservationPage: + self.queries.append(query) + return self.page + + +def _root_observation(trace_id: str, suffix: str) -> ObservationRecord: + raw = JsonDocument("{}") + return ObservationRecord( + id=ObservationId(f"observation-{suffix}"), + trace_id=TraceId(trace_id), + start_time=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + end_time=None, + project_id=ProjectId("project-1"), + parent_observation_id=None, + type=ObservationType.AGENT, + is_root=True, + name="agent", + level=None, + version=None, + environment="itsm-bench", + user_id=None, + session_id="candidate-session", + created_at=None, + updated_at=None, + metadata=None, + usage=None, + costs=None, + total_cost=None, + tags=(), + release="d" * 40, + trace_name=None, + raw=raw, + digest=Sha256Digest("sha256:" + "0" * 64), + ) + + +def _controls(policy: ExperimentPolicySnapshot) -> ExperimentControls: + return ExperimentControls( + model=policy.model, + task_ids=policy.task_ids, + benchmark_config_digest=policy.benchmark_config_digest, + verifier=policy.verifier, + environment=policy.environment, + concurrency=policy.concurrency, + max_retries=policy.max_retries, + ) + + +def _candidate_request( + tmp_path: Path, + root: Path, + hypothesis: HarnessHypothesis, +) -> CandidateExecutionInput: + benchmark = tmp_path / "benchmark" + executable = tmp_path / "harbor" + config = benchmark / "config.json" + config.write_text("{}") + return CandidateExecutionInput( + workspace_root=root, + worktree_parent=tmp_path / "candidates", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config.relative_to(benchmark), + experiment_id=hypothesis.experiment_id, + hypothesis_id=hypothesis.id.value, + ) + + +def test_candidate_id_binds_policy_hypothesis_source_tree_and_controls() -> None: + candidate = CandidateId.build( + policy_digest="sha256:" + "1" * 64, + hypothesis_id="sha256:" + "2" * 64, + source_commit="3" * 40, + candidate_tree="4" * 40, + controls_digest="sha256:" + "5" * 64, + ) + + expected = hashlib.sha256( + ( + '{"schema_version":1,"policy_digest":"sha256:' + + "1" * 64 + + '","hypothesis_id":"sha256:' + + "2" * 64 + + '","source_commit":"' + + "3" * 40 + + '","candidate_tree":"' + + "4" * 40 + + '","controls_digest":"sha256:' + + "5" * 64 + + '"}' + ).encode("utf-8") + ).hexdigest() + + assert candidate.value == f"sha256:{expected}" + assert str(candidate) == candidate.value + + +def test_candidate_id_changes_when_any_authority_input_changes() -> None: + values = { + "policy_digest": "sha256:" + "1" * 64, + "hypothesis_id": "sha256:" + "2" * 64, + "source_commit": "3" * 40, + "candidate_tree": "4" * 40, + "controls_digest": "sha256:" + "5" * 64, + } + baseline = CandidateId.build(**values) + + for field, replacement in ( + ("policy_digest", "sha256:" + "a" * 64), + ("hypothesis_id", "sha256:" + "b" * 64), + ("source_commit", "c" * 40), + ("candidate_tree", "d" * 40), + ("controls_digest", "sha256:" + "e" * 64), + ): + changed = dict(values) + changed[field] = replacement + assert CandidateId.build(**changed) != baseline + + +@pytest.mark.parametrize("trace_id", ("", "invalid trace", "x" * 257)) +def test_trace_match_rejects_invalid_trace_identifiers(trace_id: str) -> None: + with pytest.raises(CandidateFailure) as raised: + TraceMatch(trace_id=trace_id, blocker=None) + + assert raised.value.code is CandidateErrorCode.INVALID_RESULT + + +def test_candidate_input_rejects_extra_relative_and_escaped_paths(tmp_path: Path) -> None: + root, _, hypothesis = _authority(tmp_path) + request = _candidate_request(tmp_path, root, hypothesis) + payload = request.model_dump_json() + + with pytest.raises(ValidationError): + CandidateExecutionInput.model_validate_json(payload[:-1] + ',"unexpected":true}') + with pytest.raises(ValidationError): + CandidateExecutionInput.model_validate_json( + payload.replace(str(request.workspace_root), "relative-workspace") + ) + with pytest.raises(ValidationError): + CandidateExecutionInput.model_validate_json( + payload.replace('"harbor_config":"config.json"', '"harbor_config":"../config.json"') + ) + + +def test_candidate_git_gateway_isolates_validates_and_commits_one_tree(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + worktree_parent = tmp_path / "candidates" + worktree_parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, worktree_parent, policy, hypothesis) + + (workspace.worktree_path / "prompt.md").write_text("Check state before finalizing.\n") + tree = gateway.inspect(workspace, policy, hypothesis) + candidate_id = CandidateId.build( + policy_digest=candidate_policy_digest(policy), + hypothesis_id=hypothesis.id.value, + source_commit=workspace.source_commit, + candidate_tree=tree.tree_id, + controls_digest=policy.controls_digest, + ) + committed = gateway.commit(workspace, tree, candidate_id, policy.experiment_id) + + assert (root / "prompt.md").read_text() == "Original prompt.\n" + assert _git(workspace.worktree_path, "rev-parse", "HEAD^") == policy.initialization_commit + assert _git(workspace.worktree_path, "rev-parse", "HEAD^{tree}") == tree.tree_id + message = _git(workspace.worktree_path, "show", "-s", "--format=%B", committed.commit) + assert message.count("OFW-Experiment: experiment-one") == 1 + assert message.count(f"OFW-Run: {candidate_id.value}") == 1 + + +def test_candidate_git_gateway_rejects_an_empty_candidate(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.EMPTY_CANDIDATE + assert _git(workspace.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit + + +def test_candidate_git_gateway_refuses_an_existing_candidate_worktree(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + gateway.prepare(root, parent, policy, hypothesis) + + with pytest.raises(CandidateFailure) as raised: + gateway.prepare(root, parent, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.WORKTREE_EXISTS + + +def test_candidate_git_gateway_rechecks_the_tree_before_commit(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + target = workspace.worktree_path / "prompt.md" + target.write_text("first candidate\n") + tree = gateway.inspect(workspace, policy, hypothesis) + target.write_text("changed after sealing\n") + candidate_id = CandidateId.build( + policy_digest=candidate_policy_digest(policy), + hypothesis_id=hypothesis.id.value, + source_commit=workspace.source_commit, + candidate_tree=tree.tree_id, + controls_digest=policy.controls_digest, + ) + + with pytest.raises(CandidateFailure) as raised: + gateway.commit(workspace, tree, candidate_id, policy.experiment_id) + + assert raised.value.code is CandidateErrorCode.STALE_COMMIT + assert _git(workspace.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit + + +def test_candidate_git_gateway_rejects_hard_linked_targets(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + candidate_target = workspace.worktree_path / "prompt.md" + candidate_target.unlink() + os.link(root / "prompt.md", candidate_target) + candidate_target.write_text("shared mutation\n") + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.UNSAFE_PATH + + +@pytest.mark.parametrize( + ("mutation", "expected"), + ( + ("out-of-scope", CandidateErrorCode.OUT_OF_SCOPE), + ("rename", CandidateErrorCode.UNSAFE_PATH), + ("managed", CandidateErrorCode.MANAGED_PATH), + ("workspace", CandidateErrorCode.MANAGED_PATH), + ("credential", CandidateErrorCode.CREDENTIAL_PATH), + ("symlink", CandidateErrorCode.UNSAFE_PATH), + ), +) +def test_candidate_git_gateway_freezes_everything_except_exact_hypothesis_targets( + tmp_path: Path, + mutation: str, + expected: CandidateErrorCode, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + candidate = workspace.worktree_path + if mutation == "out-of-scope": + (candidate / "tools.py").write_text("unsafe = True\n") + elif mutation == "rename": + _git(candidate, "mv", "prompt.md", "renamed.md") + elif mutation == "managed": + (candidate / "PROGRAM.md").write_text("changed\n") + elif mutation == "workspace": + (candidate / ".workspace").mkdir() + (candidate / ".workspace/state.json").write_text("{}") + elif mutation == "credential": + (candidate / ".env").write_text("TOKEN=secret\n") + else: + (candidate / "prompt.md").unlink() + (candidate / "prompt.md").symlink_to(candidate / "PROGRAM.md") + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is expected + assert _git(candidate, "rev-parse", "HEAD") == policy.initialization_commit + + +@pytest.mark.parametrize("invalid", ("missing", "file")) +def test_candidate_git_gateway_rejects_invalid_workspace_roots( + tmp_path: Path, + invalid: str, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "missing" + if invalid == "file": + parent.write_text("not a directory\n") + + with pytest.raises(CandidateFailure) as raised: + CandidateGitGateway().prepare(root, parent, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.INVALID_WORKSPACE + + +@pytest.mark.parametrize("drift", ("accepted", "candidate")) +def test_candidate_git_gateway_rejects_stale_or_unrelated_ancestry( + tmp_path: Path, + drift: str, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + target = root if drift == "accepted" else workspace.worktree_path + (target / "prompt.md").write_text("changed\n") + _git(target, "add", "prompt.md") + _git(target, "commit", "-qm", "unauthorized commit") + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.STALE_COMMIT + + +@pytest.mark.parametrize("drift", ("experiment", "source", "branch", "target")) +def test_candidate_git_gateway_rejects_policy_or_hypothesis_drift( + tmp_path: Path, + drift: str, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + if drift == "experiment": + hypothesis = replace(hypothesis, experiment_id="different-experiment") + elif drift == "source": + hypothesis = replace(hypothesis, source_commit="f" * 40) + elif drift == "branch": + _git(root, "branch", "-m", "wrong-branch") + else: + hypothesis = replace( + hypothesis, + target=HarnessChangeTarget(ComponentKind.TOOL, (Path("missing.py"),)), + ) + + with pytest.raises(CandidateFailure) as raised: + CandidateGitGateway().validate_accepted(root, policy, hypothesis) + + assert raised.value.code in { + CandidateErrorCode.STALE_COMMIT, + CandidateErrorCode.STALE_POLICY, + } + + +def test_candidate_service_creates_launches_polls_and_replays_authoritative_receipts( + tmp_path: Path, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + locator = _FakeTraceLocator() + locator.cost_usd = 0.25 + outcomes = _FakeOutcomeStore() + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=locator, + outcome_store=outcomes, + ) + + editing = service.execute(request) + assert editing.status is CandidateStatus.WARNING + assert editing.phase is CandidatePhase.EDITING + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("Check state before finalizing.\n") + + running = service.execute(request) + assert running.phase is CandidatePhase.RUNNING + assert running.candidate_id is not None + assert running.candidate_commit is not None + assert len(runner.runs) == 1 + still_running = service.execute(request) + assert still_running.phase is CandidatePhase.RUNNING + assert still_running.candidate_id == running.candidate_id + assert len(runner.runs) == 1 + runner.summary = ExperimentSummary( + trials=( + ExperimentTrial( + task_id="task-1", + task_checksum="checksum-1", + exception=False, + verdict=None, + reward=1.0, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 1, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 1, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-1/result.json",), + ), + ExperimentTrial( + task_id="task-2", + task_checksum="checksum-2", + exception=False, + verdict=None, + reward=0.0, + started_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 2, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-2/result.json",), + ), + ) + ) + + complete = service.execute(request) + repeated = service.execute(request) + + assert complete == repeated + assert complete.status is CandidateStatus.SUCCESS + assert complete.phase is CandidatePhase.COMPLETE + assert complete.verifier_passes == 1 + assert complete.verifier_failures == 1 + assert complete.unverified_trials == 0 + assert tuple(receipt.task_id for receipt in complete.outcome_receipts) == ( + "task-1", + "task-2", + ) + assert complete.evaluated_run_receipt is not None + assert complete.evaluated_run_receipt.task_ids == policy.task_ids + assert complete.evaluated_run_receipt.outcome_receipts[0].cost_usd == 0.25 + assert complete.evaluated_run_receipt.outcome_receipts[0].latency_seconds == 30.0 + reloaded = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=locator, + outcome_store=outcomes, + ).execute(request) + assert reloaded.evaluated_run_receipt == complete.evaluated_run_receipt + assert len(runner.runs) == 1 + assert len(locator.requests) == 2 + assert len(outcomes.outcomes) == 2 + state = next((root / ".git/ofw/candidates").glob("*/state.json")).read_text() + assert "\n \"input\"" not in state + assert "\n \"output\"" not in state + assert "\n \"outcome_receipts\"" not in state + assert "\n \"blockers\"" not in state + assert "test-openai-key" not in state + + +@pytest.mark.parametrize( + ("trace_ids", "cursor", "trace_id", "blocker"), + ( + (("trace-1",), None, "trace-1", None), + ((), None, None, CandidateBlockerCode.TRACE_NOT_FOUND), + (("trace-1", "trace-2"), None, None, CandidateBlockerCode.TRACE_AMBIGUOUS), + (("trace-1",), "next", None, CandidateBlockerCode.TRACE_AMBIGUOUS), + ), +) +def test_trace_locator_requires_exactly_one_complete_structural_match( + trace_ids: tuple[str, ...], + cursor: str | None, + trace_id: str | None, + blocker: CandidateBlockerCode | None, +) -> None: + page = ObservationPage( + tuple(_root_observation(value, str(index)) for index, value in enumerate(trace_ids)), + None if cursor is None else PageCursor(cursor), + ) + reader = _ObservationReader(page) + locator = LangfuseCandidateTraceLocator(reader) + request = TraceMatchRequest( + task_id="task-1", + session_id="candidate-session", + environment="itsm-bench", + release="d" * 40, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + ) + + assert locator.locate(request) == TraceMatch(trace_id=trace_id, blocker=blocker) + query = reader.queries[0] + assert query.session_id == request.session_id + assert query.environment == request.environment + assert query.release == request.release + assert query.limit == 2 + assert query.is_root_observation is True + assert query.window is not None + assert query.window.start == request.started_at + assert query.window.end == request.finished_at + assert tuple(field.value for field in query.fields) == ("core", "basic", "trace_context") + + +def test_trace_locator_preserves_provider_attributed_cost() -> None: + record = replace(_root_observation("trace-1", "cost"), total_cost=0.75) + reader = _ObservationReader(ObservationPage((record,), None)) + request = TraceMatchRequest( + task_id="task-1", + session_id="candidate-session", + environment="itsm-bench", + release="d" * 40, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + ) + + assert LangfuseCandidateTraceLocator(reader).locate(request) == TraceMatch( + trace_id="trace-1", + blocker=None, + cost_usd=0.75, + ) + + +def test_candidate_service_rejects_controls_drift_before_commit_or_launch(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + drifted = ExperimentControls( + model="different-model", + task_ids=policy.task_ids, + benchmark_config_digest=policy.benchmark_config_digest, + verifier=policy.verifier, + environment=policy.environment, + concurrency=policy.concurrency, + max_retries=policy.max_retries, + ) + runner = _FakeRunner(drifted) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + + rejected = service.execute(request) + + assert rejected.error_code is CandidateErrorCode.CONTROLS_DRIFT + assert _git(editing.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit + assert runner.runs == [] + + +def test_candidate_service_sanitizes_missing_runtime_credentials(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + runner.failure = PreparationFailure( + PreparationErrorCode.MISSING_ENVIRONMENT, + "OPENAI_API_KEY|AZURE_OPENAI_API_KEY", + ) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + + rejected = service.execute(request) + + assert rejected.error_code is CandidateErrorCode.MISSING_ENVIRONMENT + assert "secret" not in rejected.summary + + +def test_candidate_service_persists_a_terminal_launch_failure(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + runner.start_failure = PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "harbor") + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + + failed = service.execute(request) + repeated = service.execute(request) + + assert failed == repeated + assert failed.phase is CandidatePhase.FAILED + assert failed.error_code is CandidateErrorCode.LAUNCH_FAILED + assert failed.candidate_commit is not None + assert runner.start_count == 1 + + +def test_candidate_service_persists_timeout_and_ignores_late_results( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + outcomes = _FakeOutcomeStore() + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=outcomes, + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + running = service.execute(request) + assert running.phase is CandidatePhase.RUNNING + + class _ExpiredDateTime(datetime): + @classmethod + def now(cls, tz: object = None) -> _ExpiredDateTime: + del tz + return cls(2100, 1, 1, tzinfo=UTC) + + monkeypatch.setattr(candidate_service_module, "datetime", _ExpiredDateTime) + timed_out = service.execute(request) + runner.summary = ExperimentSummary(()) + repeated = service.execute(request) + + assert timed_out == repeated + assert timed_out.phase is CandidatePhase.FAILED + assert timed_out.error_code is CandidateErrorCode.CANDIDATE_TIMEOUT + assert len(runner.cancelled) == 1 + assert runner.cancelled[0][1] == 123 + assert outcomes.outcomes == [] + + +def test_candidate_service_rejects_a_missing_hypothesis_receipt(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + request = _candidate_request(tmp_path, root, hypothesis) + missing = CandidateExecutionInput( + workspace_root=request.workspace_root, + worktree_parent=request.worktree_parent, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=request.harbor_config, + experiment_id=request.experiment_id, + hypothesis_id="sha256:" + "f" * 64, + ) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=_FakeRunner(_controls(policy)), + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + + rejected = service.execute(missing) + + assert rejected.error_code is CandidateErrorCode.STALE_POLICY + + +def test_candidate_service_rejects_a_reused_hypothesis_with_different_runtime_paths( + tmp_path: Path, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=_FakeRunner(_controls(policy)), + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + service.execute(request) + conflicting = CandidateExecutionInput( + workspace_root=request.workspace_root, + worktree_parent=request.worktree_parent, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=Path("different.json"), + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + ) + + rejected = service.execute(conflicting) + + assert rejected.error_code is CandidateErrorCode.REQUEST_CONFLICT + + +def test_candidate_service_keeps_unsupported_and_ambiguous_trials_unverified( + tmp_path: Path, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + locator = _FakeTraceLocator() + locator.blocker = CandidateBlockerCode.TRACE_AMBIGUOUS + outcomes = _FakeOutcomeStore() + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=locator, + outcome_store=outcomes, + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + service.execute(request) + runner.summary = ExperimentSummary( + trials=( + ExperimentTrial( + task_id="task-1", + task_checksum="checksum-1", + exception=False, + verdict=None, + reward=0.5, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 1, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 1, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-1/result.json",), + ), + ExperimentTrial( + task_id="task-2", + task_checksum="checksum-2", + exception=False, + verdict=None, + reward=0.0, + started_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 2, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-2/result.json",), + ), + ) + ) + + complete = service.execute(request) + + assert complete.outcome_receipts == () + assert complete.status is CandidateStatus.WARNING + assert complete.evaluated_run_receipt is not None + assert complete.evaluated_run_receipt.task_ids == policy.task_ids + assert tuple(item.task_id for item in complete.evaluated_run_receipt.blockers) == ( + "task-1", + "task-2", + ) + assert tuple(blocker.code for blocker in complete.blockers) == ( + CandidateBlockerCode.UNSUPPORTED_REWARD.value, + CandidateBlockerCode.TRACE_AMBIGUOUS.value, + ) + assert complete.unverified_trials == 2 + assert len(locator.requests) == 1 + assert outcomes.outcomes == [] diff --git a/tests/test_evaluated_run_receipt.py b/tests/test_evaluated_run_receipt.py new file mode 100644 index 0000000..cf34f53 --- /dev/null +++ b/tests/test_evaluated_run_receipt.py @@ -0,0 +1,217 @@ +"""Strict durable evaluated-run receipt contracts.""" + +from __future__ import annotations + +import math + +import pytest +from pydantic import TypeAdapter, ValidationError + +from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + RunSide, + VerifierVerdict, +) + +_DIGEST = "sha256:" + "a" * 64 +_COMMIT = "b" * 40 +_TREE = "c" * 40 +_JSON_OBJECT = TypeAdapter(dict[str, object]) + + +def _task( + task_id: str, + *, + verdict: VerifierVerdict = VerifierVerdict.PASS, + score: float | None = 1.0, +) -> EvaluatedTaskReceipt: + return EvaluatedTaskReceipt( + task_id=task_id, + trace_id=f"trace-{task_id}", + score_id=f"score-{task_id}", + verdict=verdict, + verifier_id="itsm-bench@checksum", + normalized_score=score, + cost_usd=0.25, + latency_seconds=1.5, + ) + + +def _blocker(task_id: str) -> EvaluatedRunBlocker: + return EvaluatedRunBlocker( + task_id=task_id, + code="trace_ambiguous", + subject="trace_mapping", + ) + + +def _receipt( + task_ids: tuple[str, ...] = ("task-1", "task-2"), + outcomes: tuple[EvaluatedTaskReceipt, ...] = (), + blockers: tuple[EvaluatedRunBlocker, ...] = (), +) -> EvaluatedRunReceipt: + return EvaluatedRunReceipt.build( + run_id="run-1", + side=RunSide.CANDIDATE, + policy_digest=_DIGEST, + controls_digest=_DIGEST, + evaluated_commit=_COMMIT, + evaluated_tree=_TREE, + task_ids=task_ids, + outcome_receipts=outcomes, + blockers=blockers, + ) + + +def test_evaluated_run_receipt_is_immutable_and_deterministic() -> None: + first = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + second = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + + assert first == second + assert first.receipt_id == first.recomputed_id() + assert first.receipt_id.startswith("sha256:") + with pytest.raises(ValidationError): + first.run_id = "other" + + +def test_evaluated_run_receipt_rejects_tampered_hash_and_extra_fields() -> None: + receipt = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + payload = _JSON_OBJECT.validate_json(receipt.model_dump_json()) + payload["run_id"] = "tampered" + with pytest.raises(ValidationError): + EvaluatedRunReceipt.model_validate(payload) + + payload = _JSON_OBJECT.validate_json(receipt.model_dump_json()) + payload["unexpected"] = True + with pytest.raises(ValidationError): + EvaluatedRunReceipt.model_validate(payload) + + +@pytest.mark.parametrize( + ("task_ids", "outcomes", "blockers"), + ( + (("task-1", "task-2"), (_task("task-1"),), ()), + (("task-1",), (_task("task-1"),), (_blocker("task-1"),)), + (("task-1",), (_task("task-1"), _task("task-1")), ()), + (("task-1",), (), (_blocker("task-1"), _blocker("task-1"))), + (("task-1",), (), (_blocker("task-2"),)), + ), +) +def test_evaluated_run_receipt_requires_an_exact_unique_partition( + task_ids: tuple[str, ...], + outcomes: tuple[EvaluatedTaskReceipt, ...], + blockers: tuple[EvaluatedRunBlocker, ...], +) -> None: + with pytest.raises((ValidationError, ValueError)): + _receipt(task_ids, outcomes, blockers) + + +def test_evaluated_run_receipt_requires_task_order_within_each_partition() -> None: + with pytest.raises((ValidationError, ValueError)): + _receipt( + task_ids=("task-1", "task-2"), + outcomes=(_task("task-2"), _task("task-1")), + ) + + +@pytest.mark.parametrize( + ("verdict", "score"), + ( + (VerifierVerdict.PASS, None), + (VerifierVerdict.PASS, 0.5), + (VerifierVerdict.FAIL, 1.0), + (VerifierVerdict.ABSTAIN, 0.0), + ), +) +def test_evaluated_task_receipt_validates_decisive_scores( + verdict: VerifierVerdict, + score: float | None, +) -> None: + with pytest.raises(ValidationError): + _task("task-1", verdict=verdict, score=score) + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("normalized_score", 2.0), + ("cost_usd", 1_000_001.0), + ("latency_seconds", 172_801.0), + ("normalized_score", math.inf), + ("cost_usd", math.inf), + ("latency_seconds", math.inf), + ), +) +def test_evaluated_task_receipt_rejects_non_finite_or_out_of_bound_metrics( + field: str, + value: float, +) -> None: + values = { + "normalized_score": 1.0, + "cost_usd": 0.25, + "latency_seconds": 1.5, + } + values[field] = value + with pytest.raises(ValidationError): + EvaluatedTaskReceipt( + task_id="task-1", + trace_id="trace-task-1", + score_id="score-task-1", + verdict=VerifierVerdict.PASS, + verifier_id="itsm-bench@checksum", + **values, + ) + + +def test_evaluated_task_receipt_preserves_missing_optional_metrics() -> None: + receipt = EvaluatedTaskReceipt( + task_id="task-1", + trace_id="trace-task-1", + score_id="score-task-1", + verdict=VerifierVerdict.ABSTAIN, + verifier_id="itsm-bench@checksum", + normalized_score=None, + cost_usd=None, + latency_seconds=None, + ) + + assert receipt.cost_usd is None + assert receipt.latency_seconds is None + + +def test_evaluated_run_receipt_rejects_non_strict_scalar_input() -> None: + payload = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + raw = _JSON_OBJECT.validate_json(payload.model_dump_json()) + raw["run_id"] = 1 + with pytest.raises(ValidationError): + EvaluatedRunReceipt.model_validate(raw) + + +def test_evaluated_run_receipt_identity_changes_with_receipt_content() -> None: + first = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + changed = EvaluatedRunReceipt.build( + run_id=first.run_id, + side=first.side, + policy_digest=first.policy_digest, + controls_digest=first.controls_digest, + evaluated_commit=first.evaluated_commit, + evaluated_tree=first.evaluated_tree, + task_ids=first.task_ids, + outcome_receipts=( + EvaluatedTaskReceipt( + task_id="task-1", + trace_id="trace-task-1", + score_id="score-task-1", + verdict=VerifierVerdict.PASS, + verifier_id="itsm-bench@checksum", + normalized_score=1.0, + cost_usd=0.5, + latency_seconds=1.5, + ), + ), + blockers=first.blockers, + ) + + assert first.receipt_id != changed.receipt_id diff --git a/tests/test_experiment_policy.py b/tests/test_experiment_policy.py new file mode 100644 index 0000000..ba522fb --- /dev/null +++ b/tests/test_experiment_policy.py @@ -0,0 +1,293 @@ +"""Canonical experiment policy publication and reload boundaries.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.preparation.contracts import ( + BaselineConfiguration, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, + build_experiment_policy, +) + + +def _git(root: Path, *arguments: str) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _repository(tmp_path: Path) -> tuple[Path, str]: + root = tmp_path / "harness" + root.mkdir() + (root / "prompt.md").write_text("Original prompt.\n", encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.name", "OpenFlywheel Test") + _git(root, "config", "user.email", "ofw@example.test") + _git(root, "add", "prompt.md") + _git(root, "commit", "-qm", "initial") + return root, _git(root, "rev-parse", "HEAD") + + +def _request( + tmp_path: Path, + root: Path, + *, + goal: str = "Improve verifier-backed quality.", +) -> PrepareWorkspaceInput: + benchmark = tmp_path / "benchmark" + benchmark.mkdir() + executable = tmp_path / "harbor" + executable.touch(mode=0o700) + return PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=root, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=Path("config.json"), + expected_task_count=2, + editable_paths=(Path("prompt.md"),), + goal=goal, + quality_target=0.9, + max_iterations=4, + no_improvement_limit=2, + max_cost_per_task_usd=1.5, + max_latency_seconds=120.0, + max_baseline_seconds=600, + ) + + +def _snapshot( + tmp_path: Path, + *, + goal: str = "Improve verifier-backed quality.", +) -> tuple[Path, ExperimentPolicySnapshot]: + tmp_path.mkdir(parents=True, exist_ok=True) + root, commit = _repository(tmp_path) + request = _request(tmp_path, root, goal=goal) + prepared = PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=root, + base_commit=commit, + initialization_commit=commit, + program_path=root / "PROGRAM.md", + ) + baseline = BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("tasks/a", "tasks/b"), + benchmark_config_digest=f"sha256:{hashlib.sha256(b'config').hexdigest()}", + verifier="itsm-bench", + environment="itsm-bench", + ) + return root, build_experiment_policy(request, prepared, baseline) + + +def _control_directory(root: Path) -> Path: + path = root / ".git/ofw/preparations/experiment-one" + path.mkdir(parents=True) + return path + + +def test_policy_is_derived_from_validated_preparation_inputs(tmp_path: Path) -> None: + _, policy = _snapshot(tmp_path) + + assert policy.experiment_id == "experiment-one" + assert policy.editable_paths == (Path("prompt.md"),) + assert policy.task_ids == ("tasks/a", "tasks/b") + assert policy.model == "openai/gpt-5.4-mini" + assert policy.verifier == "itsm-bench" + assert policy.environment == "itsm-bench" + assert policy.concurrency == 1 + assert policy.max_retries == 0 + assert policy.baseline_reused is False + assert policy.controls_digest.startswith("sha256:") + assert policy.controls_digest == policy.recomputed_controls_digest() + + +def test_maximum_valid_policy_fits_its_publication_bound(tmp_path: Path) -> None: + request = PrepareWorkspaceInput( + experiment_id="maximum", + harness_root=tmp_path / "harness", + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=tmp_path / "benchmark", + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + expected_task_count=500, + editable_paths=tuple(Path(f"{index:02d}/" + "p" * 1021) for index in range(50)), + goal="g" * 2000, + quality_target=1.0, + max_iterations=100, + no_improvement_limit=100, + max_baseline_seconds=172800, + ) + prepared = PreparedGitWorkspace( + branch_name="ofw/maximum", + worktree_path=tmp_path / "worktree", + base_commit="1" * 40, + initialization_commit="2" * 40, + program_path=tmp_path / "worktree/PROGRAM.md", + ) + baseline = BaselineConfiguration( + model="m" * 256, + task_ids=tuple(f"{index:03d}-" + "t" * 252 for index in range(500)), + benchmark_config_digest="sha256:" + "3" * 64, + verifier="v" * 128, + environment="e" * 128, + ) + policy = build_experiment_policy(request, prepared, baseline) + control = tmp_path / "common/ofw/preparations/maximum" + control.mkdir(parents=True) + + published = FileExperimentPolicyRepository().publish(control, policy) + + assert published.stat().st_size <= 256 * 1024 + + +def test_policy_schema_rejects_extra_fields_and_tampered_controls(tmp_path: Path) -> None: + _, policy = _snapshot(tmp_path) + payload = policy.model_dump_json() + extra = payload[:-1] + ',"unexpected":true}' + + with pytest.raises(ValidationError): + ExperimentPolicySnapshot.model_validate_json(extra) + + tampered = payload.replace(policy.model, "different-model") + with pytest.raises(ValidationError): + ExperimentPolicySnapshot.model_validate_json(tampered) + + escaped = payload.replace('"editable_paths":["prompt.md"]', '"editable_paths":["../x"]') + with pytest.raises(ValidationError): + ExperimentPolicySnapshot.model_validate_json(escaped) + + +@pytest.mark.parametrize( + ("field", "malformed"), + ( + ("base_commit", "x" + "1" * 40 + "y"), + ("initialization_commit", "x" + "2" * 40 + "y"), + ("benchmark_config_digest", "xsha256:" + "3" * 64 + "y"), + ), +) +def test_policy_schema_rejects_prefixed_and_suffixed_identifiers( + tmp_path: Path, + field: str, + malformed: str, +) -> None: + _, policy = _snapshot(tmp_path) + originals: dict[str, str] = { + "base_commit": policy.base_commit, + "initialization_commit": policy.initialization_commit, + "benchmark_config_digest": policy.benchmark_config_digest, + } + payload = policy.model_dump_json().replace( + f'"{field}":"{originals[field]}"', + f'"{field}":"{malformed}"', + ) + + with pytest.raises(ValidationError, match=field): + ExperimentPolicySnapshot.model_validate_json(payload) + + +def test_policy_publish_is_atomic_idempotent_and_conflict_detecting(tmp_path: Path) -> None: + root, policy = _snapshot(tmp_path) + control = _control_directory(root) + repository = FileExperimentPolicyRepository() + + first = repository.publish(control, policy) + repeated = repository.publish(control, policy) + + assert first == repeated == control / "policy.json" + assert repository.load(root, "experiment-one") == policy + _, conflicting = _snapshot(tmp_path / "conflict", goal="Conflicting goal.") + with pytest.raises(ExperimentPolicyFailure) as raised: + repository.publish(control, conflicting) + assert raised.value.code is ExperimentPolicyErrorCode.POLICY_CONFLICT + + +def test_policy_load_rejects_a_mismatched_experiment(tmp_path: Path) -> None: + root, policy = _snapshot(tmp_path) + control = root / ".git/ofw/preparations/experiment-two" + control.mkdir(parents=True) + repository = FileExperimentPolicyRepository() + repository.publish(control, policy) + + with pytest.raises(ExperimentPolicyFailure) as raised: + repository.load(root, "experiment-two") + + assert raised.value.code is ExperimentPolicyErrorCode.POLICY_INVALID + + +@pytest.mark.parametrize("kind", ("symlink", "fifo")) +def test_policy_rejects_non_regular_existing_file(tmp_path: Path, kind: str) -> None: + root, policy = _snapshot(tmp_path) + control = _control_directory(root) + path = control / "policy.json" + if kind == "symlink": + path.symlink_to(root / "prompt.md") + else: + os.mkfifo(path) + + with pytest.raises(ExperimentPolicyFailure) as raised: + FileExperimentPolicyRepository().publish(control, policy) + + assert raised.value.code is ExperimentPolicyErrorCode.POLICY_WRITE_FAILED + + +def test_policy_reload_rejects_missing_oversized_and_invalid_files(tmp_path: Path) -> None: + root, _ = _snapshot(tmp_path) + control = _control_directory(root) + repository = FileExperimentPolicyRepository() + + with pytest.raises(ExperimentPolicyFailure) as missing: + repository.load(root, "experiment-one") + assert missing.value.code is ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED + + path = control / "policy.json" + path.write_bytes(b"x" * (256 * 1024 + 1)) + with pytest.raises(ExperimentPolicyFailure) as oversized: + repository.load(root, "experiment-one") + assert oversized.value.code is ExperimentPolicyErrorCode.POLICY_TOO_LARGE + + path.write_text("{}", encoding="utf-8") + with pytest.raises(ExperimentPolicyFailure) as invalid: + repository.load(root, "experiment-one") + assert invalid.value.code is ExperimentPolicyErrorCode.POLICY_INVALID + + +def test_policy_publish_fsyncs_file_and_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root, policy = _snapshot(tmp_path) + control = _control_directory(root) + calls: list[int] = [] + original = os.fsync + + def capture(descriptor: int) -> None: + calls.append(descriptor) + original(descriptor) + + monkeypatch.setattr(os, "fsync", capture) + + FileExperimentPolicyRepository().publish(control, policy) + + assert len(calls) >= 2 diff --git a/tests/test_failure_workspace.py b/tests/test_failure_workspace.py index cc2d700..6a1886e 100644 --- a/tests/test_failure_workspace.py +++ b/tests/test_failure_workspace.py @@ -14,7 +14,6 @@ import pytest from pydantic import ValidationError -import ofw.evaluation.failure_workspace as failure_workspace_module from ofw.contracts import ComponentKind, Sha256Digest from ofw.evaluation.failure import FailureEvidenceStatus, FailureType from ofw.evaluation.failure_curation import ( @@ -188,6 +187,7 @@ def _oversized_curation() -> FailureCuration: def _expected_artifact(artifact_id: str) -> FailureArtifact: return FailureArtifact( artifact_id=artifact_id, + content_digest="sha256:e712d9d50d334533c1dd8ec75011478d588392d806a1e99633e386aede9e5ba8", trace_id="trace-1", task_id="task-1", verifier_id="itsm-bench@v1", @@ -406,71 +406,6 @@ def test_workspace_symlink_cannot_escape_the_prepared_root(tmp_path: Path) -> No assert tuple(outside.iterdir()) == () -def test_symlink_swap_after_validation_cannot_redirect_the_artifact( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - root = _prepared_workspace(tmp_path) - outside = tmp_path / "outside" - outside.mkdir() - original = failure_workspace_module._prepare_workspace_directories - - def prepare_then_swap( - prepared_root: Path, - workspace: Path, - failures: Path, - ) -> failure_workspace_module._DirectoryChainIdentity: - identity = original(prepared_root, workspace, failures) - failures.rmdir() - failures.symlink_to(outside, target_is_directory=True) - return identity - - monkeypatch.setattr( - failure_workspace_module, - "_prepare_workspace_directories", - prepare_then_swap, - ) - - with pytest.raises(FailureWorkspaceFailure): - FailureWorkspaceService(FileFailureWorkspace()).record(_request(root)) - - assert tuple(outside.iterdir()) == () - - -def test_workspace_directory_swap_after_validation_cannot_receive_the_artifact( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - root = _prepared_workspace(tmp_path) - replacement = tmp_path / "replacement" - displaced = tmp_path / "validated-workspace" - (replacement / "failures").mkdir(parents=True) - original = failure_workspace_module._prepare_workspace_directories - - def prepare_then_swap( - prepared_root: Path, - workspace: Path, - failures: Path, - ) -> failure_workspace_module._DirectoryChainIdentity: - identity = original(prepared_root, workspace, failures) - workspace.rename(displaced) - replacement.rename(workspace) - return identity - - monkeypatch.setattr( - failure_workspace_module, - "_prepare_workspace_directories", - prepare_then_swap, - ) - - with pytest.raises(FailureWorkspaceFailure) as raised: - FailureWorkspaceService(FileFailureWorkspace()).record(_request(root)) - - assert raised.value.code is FailureWorkspaceErrorCode.WRITE_FAILED - assert not tuple(displaced.rglob("*.json")) - assert not tuple((root / ".workspace").rglob("*.json")) - - def test_curates_recorded_failures_without_copying_trace_content(tmp_path: Path) -> None: root = _prepared_workspace(tmp_path) artifact_ids, _ = _record_failures(root) diff --git a/tests/test_harbor_experiment.py b/tests/test_harbor_experiment.py new file mode 100644 index 0000000..6095abf --- /dev/null +++ b/tests/test_harbor_experiment.py @@ -0,0 +1,440 @@ +"""Generalized deterministic Harbor execution for candidate and baseline runs.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import BaseModel, ConfigDict + +from ofw.preparation.contracts import ( + BaselineRun, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + PreparationErrorCode, + PreparationFailure, +) +from ofw.preparation.harbor import HarborBaselineRunner, HarborExperimentRunner + + +class _EnvironmentCapture(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + source: str + environment: str + release: str + session: str + + +def _credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "test-openai-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://example.test/openai/v1") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.test") + + +def _benchmark(tmp_path: Path) -> tuple[Path, Path, Path]: + root = tmp_path / "benchmark" + root.mkdir() + adapter = root / "agents/ofw_hermes.py" + adapter.parent.mkdir() + adapter.write_text('SOURCE = "OFW_HERMES_SOURCE"\n') + config = root / "config.json" + config.write_text( + """{ + "agents": [{ + "name": "agents.ofw_hermes:OfwHermes", + "model_name": "openai/gpt-5.4-mini" + }], + "tasks": [{"path": "task-1"}, {"path": "task-2"}] +} +""" + ) + executable = tmp_path / "harbor" + executable.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +args = sys.argv[1:] +name = args[args.index("--job-name") + 1] +root = Path(args[args.index("--jobs-dir") + 1]) / name +root.mkdir(parents=True) +(root / "environment.json").write_text(json.dumps({ + "source": os.environ["OFW_HERMES_SOURCE"], + "environment": os.environ["HERMES_LANGFUSE_ENV"], + "release": os.environ["HERMES_LANGFUSE_RELEASE"], + "session": os.environ["HERMES_LANGFUSE_SESSION_ID"], +})) +for index, reward in enumerate((1.0, 0.0), start=1): + trial = root / f"task-{index}__trial" + (trial / "verifier").mkdir(parents=True) + (trial / "result.json").write_text(json.dumps({ + "task_name": f"display-{index}", + "task_id": {"path": f"task-{index}"}, + "task_checksum": f"checksum-{index}", + "exception_info": None, + "agent_execution": { + "started_at": f"2026-09-02T10:0{index}:00Z", + "finished_at": f"2026-09-02T10:0{index}:30Z" + }, + "verifier": {"finished_at": f"2026-09-02T10:0{index}:31Z"}, + "verifier_result": {"rewards": {"reward": reward}} + })) +(root / "result.json").write_text(json.dumps({ + "finished_at": "2026-09-02T10:03:00Z", + "n_total_trials": 2 +})) +""" + ) + executable.chmod(0o755) + return root, executable, config + + +def _cancel_run(tmp_path: Path) -> ExperimentRun: + return ExperimentRun( + run_id="candidate-one", + benchmark_root=tmp_path, + harbor_executable=Path(sys.executable), + harbor_config=tmp_path / "config.json", + job_path=tmp_path / "jobs/candidate-one", + log_path=tmp_path / "candidate.log", + source_root=tmp_path, + release="a" * 40, + session_id="candidate-session", + controls=ExperimentControls( + model="model", + task_ids=("task-1",), + benchmark_config_digest="sha256:" + "b" * 64, + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ), + ) + + +def test_generalized_harbor_runner_freezes_controls_environment_and_trials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/candidate-one", + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="sha256-" + "b" * 64, + controls=controls, + ) + pid = runner.start(run) + waited, status = os.waitpid(pid, 0) + summary = runner.summarize(run) + + assert controls == ExperimentControls( + model="openai/gpt-5.4-mini", + task_ids=("task-1", "task-2"), + benchmark_config_digest=controls.benchmark_config_digest, + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ) + assert waited == pid + assert os.waitstatus_to_exitcode(status) == 0 + assert isinstance(summary, ExperimentSummary) + assert tuple(trial.task_id for trial in summary.trials) == ("task-1", "task-2") + assert summary.trials[0].reward == 1.0 + assert summary.trials[1].reward == 0.0 + assert summary.trials[0].started_at == datetime(2026, 9, 2, 10, 1, tzinfo=UTC) + assert summary.trials[0].evidence == ( + "harbor://candidate-one/task-1__trial/result.json", + "harbor://candidate-one/task-1__trial/verifier", + ) + environment = _EnvironmentCapture.model_validate_json( + (run.job_path / "environment.json").read_text() + ) + assert environment == _EnvironmentCapture( + source=str(source), + environment="itsm-bench", + release="a" * 40, + session="sha256-" + "b" * 64, + ) + + +def test_generalized_harbor_runner_rechecks_controls_immediately_before_launch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/candidate-one", + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="sha256-" + "b" * 64, + controls=controls, + ) + config.write_text(config.read_text().replace("gpt-5.4-mini", "different-model")) + + with pytest.raises(PreparationFailure) as raised: + runner.start(run) + + assert raised.value.code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert not run.job_path.exists() + + +def test_harbor_rejects_existing_jobs_and_baseline_launch_recomputes_controls( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + experiment = HarborExperimentRunner() + controls = experiment.validate(benchmark, executable, config.relative_to(benchmark)) + job = benchmark / "jobs/candidate-one" + job.mkdir(parents=True) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=job, + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="candidate-session", + controls=controls, + ) + + with pytest.raises(PreparationFailure) as existing: + experiment.start(run) + baseline_run = BaselineRun( + experiment_id="baseline", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/baseline", + log_path=tmp_path / "baseline.log", + worktree_path=source, + initialization_commit="a" * 40, + controls=controls, + ) + pid = HarborBaselineRunner().start(baseline_run) + waited, status = os.waitpid(pid, 0) + + assert existing.value.code is PreparationErrorCode.LAUNCH_FAILED + assert waited == pid + assert os.waitstatus_to_exitcode(status) == 0 + assert baseline_run.job_path.is_dir() + + +def test_baseline_launch_rejects_controls_changed_after_the_persisted_snapshot( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + controls = HarborExperimentRunner().validate( + benchmark, + executable, + config.relative_to(benchmark), + ) + run = BaselineRun( + experiment_id="baseline", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/baseline", + log_path=tmp_path / "baseline.log", + worktree_path=source, + initialization_commit="a" * 40, + controls=controls, + ) + config.write_text(config.read_text().replace("gpt-5.4-mini", "different-model")) + + with pytest.raises(PreparationFailure) as raised: + HarborBaselineRunner().start(run) + + assert raised.value.code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert not run.job_path.exists() + + +def test_generalized_harbor_cancel_terminates_and_reaps_the_process_group( + tmp_path: Path, +) -> None: + process = subprocess.Popen( # nosec B603 + (sys.executable, "-c", "import time; time.sleep(30)"), + start_new_session=True, + ) + run = _cancel_run(tmp_path) + try: + HarborExperimentRunner().cancel(run, process.pid) + return_code = process.wait(timeout=5) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + assert return_code < 0 + + +def test_generalized_harbor_cancel_handles_absent_and_failed_processes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runner = HarborExperimentRunner() + run = _cancel_run(tmp_path) + runner.cancel(run, None) + + def missing(process_id: int, signal_number: int) -> None: + del process_id, signal_number + raise ProcessLookupError + + monkeypatch.setattr(os, "killpg", missing) + runner.cancel(run, 123) + + def denied(process_id: int, signal_number: int) -> None: + del process_id, signal_number + raise PermissionError + + monkeypatch.setattr(os, "killpg", denied) + with pytest.raises(PreparationFailure) as raised: + runner.cancel(run, 123) + + assert raised.value.code is PreparationErrorCode.LAUNCH_FAILED + + +@pytest.mark.parametrize( + ("started_at", "finished_at", "evaluated_at"), + ( + ( + "2026-09-02T10:01:00", + "2026-09-02T10:01:30Z", + "2026-09-02T10:01:31Z", + ), + ( + "2026-09-02T10:01:30Z", + "2026-09-02T10:01:00Z", + "2026-09-02T10:01:31Z", + ), + ( + "2026-09-02T10:01:00Z", + "2026-09-02T10:01:30Z", + "2026-09-02T10:01:29Z", + ), + ), +) +def test_generalized_harbor_runner_maps_invalid_trial_time_to_typed_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + started_at: str, + finished_at: str, + evaluated_at: str, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + job = benchmark / "jobs/candidate-one" + trial = job / "task-1__trial" + trial.mkdir(parents=True) + (job / "result.json").write_text('{"finished_at":"2026-09-02T10:03:00Z","n_total_trials":1}') + (trial / "result.json").write_text( + f"""{{ + "task_id": "task-1", + "task_name": "display name", + "task_checksum": "checksum-1", + "exception_info": null, + "agent_execution": {{"started_at": "{started_at}", "finished_at": "{finished_at}"}}, + "verifier": {{"finished_at": "{evaluated_at}"}}, + "verifier_result": {{"rewards": {{"reward": 1.0}}}} +}}""" + ) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=job, + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="candidate-session", + controls=controls, + ) + + with pytest.raises(PreparationFailure) as raised: + runner.summarize(run) + + assert raised.value.code is PreparationErrorCode.INVALID_BASELINE_RESULT + assert raised.value.subject == "trial timestamps" + + +def test_generalized_harbor_runner_rejects_trials_without_mapping_fields( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + job = benchmark / "jobs/candidate-one" + trial = job / "task-1__trial" + trial.mkdir(parents=True) + (job / "result.json").write_text('{"finished_at":"2026-09-02T10:03:00Z","n_total_trials":1}') + (trial / "result.json").write_text( + '{"exception_info":null,"verifier_result":{"rewards":{"reward":1.0}}}' + ) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=job, + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="candidate-session", + controls=controls, + ) + + with pytest.raises(PreparationFailure) as raised: + runner.summarize(run) + + assert raised.value.code is PreparationErrorCode.INVALID_BASELINE_RESULT diff --git a/tests/test_harbor_preparation.py b/tests/test_harbor_preparation.py index 9fa3abe..db4c8fd 100644 --- a/tests/test_harbor_preparation.py +++ b/tests/test_harbor_preparation.py @@ -7,6 +7,7 @@ from ofw.preparation import ( BaselineRun, BaselineSummary, + ExperimentControls, PreparationErrorCode, PreparationFailure, ) @@ -23,6 +24,15 @@ def _run(tmp_path: Path, job_path: Path) -> BaselineRun: log_path=tmp_path / "baseline.log", worktree_path=tmp_path / "worktree", initialization_commit="0" * 40, + controls=ExperimentControls( + model="openai/gpt-5.4-mini", + task_ids=(), + benchmark_config_digest="sha256:" + "0" * 64, + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ), ) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py new file mode 100644 index 0000000..d8070c6 --- /dev/null +++ b/tests/test_hypothesis.py @@ -0,0 +1,807 @@ +"""Evidence-backed hypothesis contracts, authority, and persistence.""" + +from __future__ import annotations + +import os +import subprocess +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.contracts import ComponentKind +from ofw.evaluation.failure import FailureEvidenceStatus, FailureType +from ofw.evaluation.failure_curation import ( + FailureCurationArtifact, + FailureCurationService, + FailureGroupInput, + RecordFailureCurationInput, +) +from ofw.evaluation.failure_patterns import FailurePatternMiningService, failure_pattern_id +from ofw.evaluation.failure_workspace import ( + FailedOutcomeInput, + FailureWorkspaceService, + FileFailureCurationWorkspace, + FileFailureWorkspace, + RecordFailureInput, +) +from ofw.evolution.hypothesis import ( + FailurePatternReferenceInput, + HarnessChangeTargetInput, + HypothesisErrorCode, + HypothesisFailure, + HypothesisService, + HypothesisStatus, + RecordHypothesisInput, +) +from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.preparation.contracts import ( + BaselineConfiguration, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import FileExperimentPolicyRepository, build_experiment_policy + +_ROOT_CAUSE = "The agent finalizes before checking the updated incident state." + + +def _git(root: Path, *arguments: str, check: bool = True) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=check, + capture_output=True, + text=True, + ).stdout.strip() + + +def _workspace(tmp_path: Path) -> tuple[Path, str]: + root = tmp_path / "harness" + root.mkdir() + (root / "prompt.md").write_text("Check state before finalizing.\n", encoding="utf-8") + (root / "tools.py").write_text("def check_state():\n return True\n", encoding="utf-8") + (root / "PROGRAM.md").write_text("# Program\n", encoding="utf-8") + (root / "experiment_config.yaml").write_text("benchmark: itsm-bench\n", encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.name", "OpenFlywheel Test") + _git(root, "config", "user.email", "ofw@example.test") + _git(root, "add", ".") + _git(root, "commit", "-qm", "prepared experiment") + _git(root, "branch", "-m", "ofw/experiment-one") + commit = _git(root, "rev-parse", "HEAD") + _publish_policy(tmp_path, root, commit) + return root, commit + + +def _publish_policy(tmp_path: Path, root: Path, commit: str) -> None: + benchmark = tmp_path / "benchmark" + benchmark.mkdir(exist_ok=True) + executable = tmp_path / "harbor" + executable.touch(mode=0o700, exist_ok=True) + request = PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=root, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=Path("config.json"), + expected_task_count=2, + editable_paths=(Path("prompt.md"),), + goal="Improve verifier-backed quality.", + quality_target=1.0, + max_iterations=3, + no_improvement_limit=2, + max_baseline_seconds=600, + ) + prepared = PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=root, + base_commit=commit, + initialization_commit=commit, + program_path=root / "PROGRAM.md", + ) + baseline = BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-1", "task-2"), + benchmark_config_digest="sha256:" + "1" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ) + policy = build_experiment_policy(request, prepared, baseline) + control = root / ".git/ofw/preparations/experiment-one" + control.mkdir(parents=True) + FileExperimentPolicyRepository().publish(control, policy) + + +def _diagnosis( + root: Path, + suffix: str, + *, + root_cause: str = _ROOT_CAUSE, + supported: bool = True, +) -> str: + evidence_status = ( + FailureEvidenceStatus.SUPPORTED if supported else FailureEvidenceStatus.INCONCLUSIVE + ) + observation = FailureWorkspaceService(FileFailureWorkspace()).record( + RecordFailureInput( + workspace_root=root, + outcome=FailedOutcomeInput( + trace_id=f"trace-{suffix}", + task_id=f"task-{suffix}", + verifier_id="itsm-bench", + evaluated_at=datetime(2026, 9, 1, 12, int(suffix), tzinfo=UTC), + score=0.0, + evidence=(f"score://{suffix}",), + outcome_score_id=f"score-{suffix}", + ), + evidence_status=evidence_status, + issue_type=FailureType.CONTROL_FLOW_FAILURE if supported else None, + expected_outcome="The incident is closed.", + actual_outcome="The incident remains open.", + critical_observation_id=f"observation-{suffix}" if supported else None, + evidence_observation_ids=(f"observation-{suffix}",) if supported else (), + root_cause=root_cause if supported else None, + counterfactual_action="Read state before finalizing." if supported else None, + inconclusive_reason=None if supported else "The decisive span is unavailable.", + ) + ) + return observation.artifact_id + + +def _service() -> HypothesisService: + workspace = FileFailureWorkspace() + return HypothesisService( + pattern_miner=FailurePatternMiningService(workspace), + repository=FileHypothesisRepository(), + ) + + +def _curation(root: Path, artifacts: tuple[str, ...]) -> tuple[str, str]: + observation = FailureCurationService(FileFailureCurationWorkspace()).record( + RecordFailureCurationInput( + workspace_root=root, + source_artifact_ids=artifacts, + groups=( + FailureGroupInput( + pattern_key="premature-completion", + title="Premature completion", + mechanism="The agent finalizes before verification.", + prevention="Require verification before completion.", + target_component=ComponentKind.PROMPT, + failure_artifact_ids=artifacts, + ), + ), + deferred=(), + ) + ) + artifact = FailureCurationArtifact.model_validate_json( + (root / observation.relative_path).read_text(encoding="utf-8") + ) + return observation.curation_id, artifact.groups[0].group_id + + +def _request( + root: Path, + commit: str, + artifacts: tuple[str, ...], + *, + curation_artifacts: tuple[str, ...] | None = None, +) -> RecordHypothesisInput: + selected_curation_artifacts = curation_artifacts or artifacts + if len(selected_curation_artifacts) >= 2: + curation_id, curation_group_id = _curation(root, selected_curation_artifacts) + else: + curation_id = "00000000-0000-0000-0000-000000000010" + curation_group_id = "00000000-0000-0000-0000-000000000011" + return RecordHypothesisInput( + workspace_root=root, + experiment_id="experiment-one", + source_commit=commit, + curation_id=curation_id, + curation_group_id=curation_group_id, + predicted_task_ids=("task-1", "task-2"), + at_risk_task_ids=("task-3",), + patterns=( + FailurePatternReferenceInput( + pattern_id=failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + _ROOT_CAUSE, + ), + diagnosis_artifact_ids=artifacts, + ), + ), + statement="Require a state check before the agent finalizes.", + rationale="Both supported failures finalize immediately after mutation.", + target=HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"),), + ), + expected_effect="The agent verifies task completion before returning success.", + regression_risks=("The extra check may increase latency.",), + ) + + +def _with_patterns( + request: RecordHypothesisInput, + patterns: tuple[FailurePatternReferenceInput, ...], +) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, + patterns=patterns, + statement=request.statement, + rationale=request.rationale, + target=request.target, + expected_effect=request.expected_effect, + regression_risks=request.regression_risks, + ) + + +def _with_target( + request: RecordHypothesisInput, + target: HarnessChangeTargetInput, +) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, + patterns=request.patterns, + statement=request.statement, + rationale=request.rationale, + target=target, + expected_effect=request.expected_effect, + regression_risks=request.regression_risks, + ) + + +def _with_curation_group( + request: RecordHypothesisInput, + curation_group_id: str, +) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, + patterns=request.patterns, + statement=request.statement, + rationale=request.rationale, + target=request.target, + expected_effect=request.expected_effect, + regression_risks=request.regression_risks, + ) + + +def _evidence_case( + case: str, + first: str, + second: str, + other: str, + inconclusive: str, +) -> tuple[str, ...]: + if case == "missing": + return first, "00000000-0000-0000-0000-000000000000" + if case == "extra": + return first, second, other + if case == "misassigned": + return first, other + return first, inconclusive + + +def test_record_hypothesis_recomputes_evidence_and_is_deterministic(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + + first = _service().record(request) + repeated = _service().record(request) + + assert first == repeated + assert first.status is HypothesisStatus.SUCCESS + assert first.source_commit == commit + assert first.curation_id == request.curation_id + assert first.curation_group_id == request.curation_group_id + assert first.pattern_count == 1 + assert first.diagnosis_count == 2 + assert first.target_paths == (Path("prompt.md"),) + assert first.relative_path == Path(f".workspace/hypotheses/{first.hypothesis_id}.json") + assert "stop" in first.next_actions[0].lower() + assert _git(root, "status", "--short") == "" + + +def test_hypothesis_identity_canonicalizes_declared_order(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + reversed_request = _with_patterns( + request, + ( + FailurePatternReferenceInput( + pattern_id=request.patterns[0].pattern_id, + diagnosis_artifact_ids=tuple(reversed(artifacts)), + ), + ), + ) + + assert _service().record(reversed_request) == _service().record(request) + + +@pytest.mark.parametrize("case", ("missing", "extra", "misassigned", "inconclusive")) +def test_hypothesis_fails_closed_on_inexact_evidence(tmp_path: Path, case: str) -> None: + root, commit = _workspace(tmp_path) + first = _diagnosis(root, "1") + second = _diagnosis(root, "2") + other = _diagnosis(root, "3", root_cause="The tool result is ignored after mutation.") + inconclusive = _diagnosis(root, "4", supported=False) + artifacts = _evidence_case(case, first, second, other, inconclusive) + curation_artifacts = (first, other) if case == "misassigned" else (first, second) + request = _request(root, commit, artifacts, curation_artifacts=curation_artifacts) + if case == "misassigned": + request = _with_patterns( + request, + ( + FailurePatternReferenceInput( + pattern_id=failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + _ROOT_CAUSE, + ), + diagnosis_artifact_ids=(other,), + ), + FailurePatternReferenceInput( + pattern_id=failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + "The tool result is ignored after mutation.", + ), + diagnosis_artifact_ids=(first,), + ), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + expected = ( + HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH + if case == "misassigned" + else HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH + ) + assert raised.value.code is expected + + +@pytest.mark.parametrize( + "drift", + ("commit", "dirty", "untracked", "branch", "target-symlink"), +) +def test_hypothesis_rejects_stale_or_drifted_workspace(tmp_path: Path, drift: str) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + if drift == "commit": + (root / "tools.py").write_text("# changed\n", encoding="utf-8") + _git(root, "add", "tools.py") + _git(root, "commit", "-qm", "move head") + elif drift == "dirty": + (root / "prompt.md").write_text("dirty\n", encoding="utf-8") + elif drift == "untracked": + (root / "untracked.txt").write_text("not prepared\n", encoding="utf-8") + elif drift == "branch": + _git(root, "branch", "-m", "wrong-branch") + else: + (root / "prompt.md").unlink() + (root / "prompt.md").symlink_to(root / "tools.py") + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code in { + HypothesisErrorCode.STALE_COMMIT, + HypothesisErrorCode.DIRTY_WORKSPACE, + HypothesisErrorCode.STALE_POLICY, + HypothesisErrorCode.INVALID_TARGET, + } + + +def test_hypothesis_rejects_semantically_tampered_diagnosis(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + path = root / ".workspace/failures" / f"{artifacts[0]}.json" + content = path.read_text(encoding="utf-8") + path.write_text( + content.replace( + "The incident remains open.", + "The incident was altered after recording.", + ), + encoding="utf-8", + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH + + +@pytest.mark.parametrize("path", (Path("tools.py"), Path("PROGRAM.md"), Path("prompt.md/child"))) +def test_hypothesis_uses_exact_editable_allowlist_and_freezes_everything_else( + tmp_path: Path, + path: Path, +) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + request = _with_target( + request, + HarnessChangeTargetInput( + component_kind=ComponentKind.TOOL, + relative_paths=(path,), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.TARGET_NOT_EDITABLE + + +def test_hypothesis_input_rejects_duplicates_escapes_empty_and_extra_fields(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifact = _diagnosis(root, "1") + payload = _request(root, commit, (artifact,)).model_dump_json() + extra = payload[:-1] + ',"unexpected":true}' + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(extra) + + escaped = payload.replace('"relative_paths":["prompt.md"]', '"relative_paths":["../prompt.md"]') + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(escaped) + + empty = payload.replace('"relative_paths":["prompt.md"]', '"relative_paths":[]') + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(empty) + + duplicate = payload.replace( + f'"diagnosis_artifact_ids":["{artifact}"]', + f'"diagnosis_artifact_ids":["{artifact}","{artifact}"]', + ) + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(duplicate) + + request = _request(root, commit, (artifact,)) + with pytest.raises(ValidationError): + _with_patterns(request, (request.patterns[0], request.patterns[0])) + with pytest.raises(ValidationError): + HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"), Path("prompt.md")), + ) + with pytest.raises(ValidationError): + RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, + patterns=request.patterns, + statement=request.statement, + rationale=request.rationale, + target=request.target, + expected_effect=request.expected_effect, + regression_risks=("same risk", "same risk"), + ) + + +@pytest.mark.parametrize( + ("field", "malformed"), + ( + ("experiment_id", "Xexperiment-one!"), + ("source_commit", "x" + "1" * 40 + "y"), + ("curation_id", "x00000000-0000-0000-0000-000000000001y"), + ("curation_group_id", "x00000000-0000-0000-0000-000000000002y"), + ), +) +def test_hypothesis_input_rejects_prefixed_and_suffixed_identifiers( + tmp_path: Path, + field: str, + malformed: str, +) -> None: + root, commit = _workspace(tmp_path) + artifact = _diagnosis(root, "1") + request = _request(root, commit, (artifact,)) + originals: dict[str, str] = { + "experiment_id": request.experiment_id, + "source_commit": request.source_commit, + "curation_id": request.curation_id, + "curation_group_id": request.curation_group_id, + } + payload = request.model_dump_json().replace( + f'"{field}":"{originals[field]}"', + f'"{field}":"{malformed}"', + ) + + with pytest.raises(ValidationError, match=field): + RecordHypothesisInput.model_validate_json(payload) + + +@pytest.mark.parametrize( + ("field", "malformed"), + ( + ("pattern_id", "xsha256:" + "1" * 64 + "y"), + ("diagnosis_artifact_ids", ("x00000000-0000-0000-0000-000000000001y",)), + ), +) +def test_pattern_reference_rejects_prefixed_and_suffixed_identifiers( + field: str, + malformed: str | tuple[str, ...], +) -> None: + payload: dict[str, str | tuple[str, ...]] = { + "pattern_id": "sha256:" + "1" * 64, + "diagnosis_artifact_ids": ("00000000-0000-0000-0000-000000000001",), + } + payload[field] = malformed + + with pytest.raises(ValidationError): + FailurePatternReferenceInput.model_validate(payload) + + +def test_hypothesis_input_accepts_its_exact_global_maximum() -> None: + artifacts = tuple(f"00000000-0000-0000-0000-{index:012x}" for index in range(50)) + request = RecordHypothesisInput( + workspace_root=Path("/prepared"), + experiment_id="maximum", + source_commit="1" * 40, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), + patterns=( + FailurePatternReferenceInput( + pattern_id="sha256:" + "1" * 64, + diagnosis_artifact_ids=artifacts, + ), + ), + statement="s" * 4000, + rationale="r" * 4000, + target=HarnessChangeTargetInput( + component_kind=ComponentKind.SKILL, + relative_paths=tuple(Path(f"skills/{index}.md") for index in range(50)), + ), + expected_effect="e" * 4000, + regression_risks=tuple(f"risk-{index}" for index in range(10)), + ) + + assert len(request.patterns[0].diagnosis_artifact_ids) == 50 + assert len(request.target.relative_paths) == 50 + assert len(request.regression_risks) == 10 + + +def test_hypothesis_predictions_must_be_disjoint() -> None: + with pytest.raises(ValidationError): + RecordHypothesisInput( + workspace_root=Path("/prepared"), + experiment_id="experiment-one", + source_commit="1" * 40, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-1",), + patterns=( + FailurePatternReferenceInput( + pattern_id="sha256:" + "1" * 64, + diagnosis_artifact_ids=("00000000-0000-0000-0000-000000000001",), + ), + ), + statement="s", + rationale="r", + target=HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"),), + ), + expected_effect="e", + regression_risks=(), + ) + + +def test_hypothesis_rejects_an_incomplete_curation_group(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + first = _diagnosis(root, "1") + second = _diagnosis(root, "2") + request = _request(root, commit, (first, second)) + incomplete = _with_patterns( + request, + ( + FailurePatternReferenceInput( + pattern_id=request.patterns[0].pattern_id, + diagnosis_artifact_ids=(first,), + ), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(incomplete) + + assert raised.value.code is HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH + + +def test_hypothesis_rejects_a_missing_curation_group(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _with_curation_group( + _request(root, commit, artifacts), + "00000000-0000-0000-0000-000000000099", + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.CURATION_GROUP_NOT_FOUND + + +def test_hypothesis_rejects_a_target_outside_the_curated_component(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _with_target( + _request(root, commit, artifacts), + HarnessChangeTargetInput( + component_kind=ComponentKind.TOOL, + relative_paths=(Path("prompt.md"),), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH + + +def test_hypothesis_rejects_a_tampered_curation_receipt(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + path = root / ".workspace/failure-curations" / f"{request.curation_id}.json" + content = path.read_text(encoding="utf-8") + path.write_text( + content.replace( + "The agent finalizes before verification.", + "The curation was changed after recording.", + ), + encoding="utf-8", + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.CURATION_INVALID + + +def test_hypothesis_missing_policy_is_typed(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + (root / ".git/ofw/preparations/experiment-one/policy.json").unlink() + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(_request(root, commit, artifacts)) + + assert raised.value.code is HypothesisErrorCode.POLICY_SNAPSHOT_REQUIRED + + +def test_hypothesis_concurrent_identical_writes_are_idempotent(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = tuple(executor.map(lambda _: _service().record(request), range(2))) + + assert results[0] == results[1] + + +def test_hypothesis_conflicting_existing_artifact_is_rejected(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + recorded = _service().record(request) + path = root / recorded.relative_path + path.write_text("{}", encoding="utf-8") + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.HYPOTHESIS_CONFLICT + + +def test_hypothesis_reload_rejects_content_tampering_under_the_original_id( + tmp_path: Path, +) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + recorded = _service().record(request) + path = root / recorded.relative_path + path.write_text( + path.read_text(encoding="utf-8").replace( + request.statement, + "A different candidate target authority.", + ), + encoding="utf-8", + ) + + with pytest.raises(HypothesisFailure) as raised: + FileHypothesisRepository().load(root, recorded.hypothesis_id) + + assert raised.value.code is HypothesisErrorCode.STALE_POLICY + + +@pytest.mark.parametrize("kind", ("symlink", "fifo")) +def test_hypothesis_rejects_non_regular_artifact_target(tmp_path: Path, kind: str) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + recorded = _service().record(request) + path = root / recorded.relative_path + path.unlink() + if kind == "symlink": + path.symlink_to(root / "prompt.md") + else: + os.mkfifo(path) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.WRITE_FAILED + + +@pytest.mark.parametrize( + "invalid", + ("missing-root", "missing-marker", "missing-target", "directory"), +) +def test_hypothesis_repository_rejects_invalid_workspace_objects( + tmp_path: Path, + invalid: str, +) -> None: + root, commit = _workspace(tmp_path) + repository = FileHypothesisRepository() + policy = repository.load_policy(root, "experiment-one") + target = Path("prompt.md") + if invalid == "missing-root": + root = tmp_path / "missing" + elif invalid == "missing-marker": + (root / "PROGRAM.md").unlink() + elif invalid == "missing-target": + target = Path("missing.md") + else: + target = Path("directory") + (root / target).mkdir() + + with pytest.raises(HypothesisFailure) as raised: + repository.validate_workspace(root, policy, commit, (target,)) + + assert raised.value.code in { + HypothesisErrorCode.STALE_POLICY, + HypothesisErrorCode.INVALID_TARGET, + } + + +def test_hypothesis_repository_sanitizes_git_failure(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + repository = FileHypothesisRepository() + policy = repository.load_policy(root, "experiment-one") + (root / ".git").rename(root / ".git-moved") + + with pytest.raises(HypothesisFailure) as raised: + repository.validate_workspace(root, policy, commit, (Path("prompt.md"),)) + + assert raised.value.code is HypothesisErrorCode.STALE_POLICY + assert "fatal" not in str(raised.value).lower() diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index ca37c49..5d931df 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -4,6 +4,7 @@ import asyncio import importlib +from contextlib import AbstractContextManager from datetime import UTC, datetime from pathlib import Path from typing import Protocol, cast @@ -51,6 +52,19 @@ VerifierId, VerifierVerdict, ) +from ofw.evolution import ( + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateExecutionService, + CandidatePhase, + CandidateStatus, + FailurePatternReferenceInput, + HarnessChangeTargetInput, + HypothesisObservation, + HypothesisService, + HypothesisStatus, + RecordHypothesisInput, +) from ofw.observability.langfuse.domain import ScoreId, TraceId from ofw.observability.langfuse.trace_query import ( GetSpanContextInput, @@ -71,6 +85,10 @@ _FAILURE_ARTIFACT_ID = "00000000-0000-0000-0000-000000000001" _SECOND_FAILURE_ARTIFACT_ID = "00000000-0000-0000-0000-000000000002" _CURATION_ID = "00000000-0000-0000-0000-000000000003" +_CURATION_GROUP_ID = "00000000-0000-0000-0000-000000000004" +_PATTERN_ID = "sha256:" + "1" * 64 +_HYPOTHESIS_ID = "sha256:" + "2" * 64 +_COMMIT = "1" * 40 _JSON_OBJECT_ADAPTER: TypeAdapter[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]) @@ -84,6 +102,10 @@ def _failure_service(self) -> FailureWorkspaceService: ... def _curation_service(self) -> FailureCurationService: ... + def _hypothesis_service(self) -> HypothesisService: ... + + def _candidate_service(self) -> AbstractContextManager[CandidateExecutionService]: ... + def _program_template(self, name: str) -> str: ... def prepare_workspace( @@ -140,6 +162,13 @@ def record_failure_curation( request: RecordFailureCurationInput, ) -> FailureCurationObservation: ... + def record_hypothesis(self, request: RecordHypothesisInput) -> HypothesisObservation: ... + + def execute_candidate( + self, + request: CandidateExecutionInput, + ) -> CandidateExecutionObservation: ... + class _FakeOutcomeStore: def __init__(self) -> None: @@ -192,6 +221,24 @@ def record(self, request: RecordFailureCurationInput) -> FailureCurationObservat return self.observation +class _FakeHypothesisService: + def __init__(self, observation: HypothesisObservation) -> None: + self.observation = observation + self.requests: list[RecordHypothesisInput] = [] + + def record(self, request: RecordHypothesisInput) -> HypothesisObservation: + self.requests.append(request) + return self.observation + + +class _FakeTraceClient: + def __init__(self) -> None: + self.close_count = 0 + + def close(self) -> None: + self.close_count += 1 + + def _module() -> OpenFlywheelMcpModule: return cast(OpenFlywheelMcpModule, importlib.import_module("ofw.mcp")) @@ -310,6 +357,53 @@ def _curation_observation() -> FailureCurationObservation: ) +def _hypothesis_request(root: Path) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=root, + experiment_id="experiment-one", + source_commit=_COMMIT, + curation_id=_CURATION_ID, + curation_group_id=_CURATION_GROUP_ID, + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), + patterns=( + FailurePatternReferenceInput( + pattern_id=_PATTERN_ID, + diagnosis_artifact_ids=(_FAILURE_ARTIFACT_ID,), + ), + ), + statement="Require a final state check.", + rationale="The supported pattern shows premature finalization.", + target=HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"),), + ), + expected_effect="The agent verifies success before finalizing.", + regression_risks=("The extra check may add latency.",), + ) + + +def _hypothesis_observation() -> HypothesisObservation: + relative_path = Path(f".workspace/hypotheses/{_HYPOTHESIS_ID}.json") + return HypothesisObservation( + status=HypothesisStatus.SUCCESS, + summary="Recorded one evidence-backed hypothesis for the prepared experiment.", + next_actions=("Stop before candidate editing and retain this hypothesis receipt.",), + artifacts=(str(relative_path), _HYPOTHESIS_ID), + hypothesis_id=_HYPOTHESIS_ID, + experiment_id="experiment-one", + source_commit=_COMMIT, + curation_id=_CURATION_ID, + curation_group_id=_CURATION_GROUP_ID, + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), + relative_path=relative_path, + pattern_count=1, + diagnosis_count=1, + target_paths=(Path("prompt.md"),), + ) + + def test_mcp_exposes_scoped_read_and_recording_tools() -> None: tools = asyncio.run(_server().list_tools()) @@ -323,6 +417,8 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: "record_failure", "mine_failure_patterns", "record_failure_curation", + "record_hypothesis", + "execute_candidate", ] assert tuple(map(_annotation_flags, tools)) == ( (False, False, True), @@ -334,6 +430,8 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: (False, False, True), (True, False, True), (False, False, True), + (False, False, True), + (False, False, True), ) @@ -633,3 +731,73 @@ def test_record_failure_curation_accepts_mcp_json_mapping(tmp_path: Path) -> Non request = _curation_request(tmp_path) assert RecordFailureCurationInput.model_validate(_json_request(request)) == request + + +def test_record_hypothesis_passes_one_strict_object_to_the_service( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _module() + expected = _hypothesis_observation() + service = _FakeHypothesisService(expected) + request = _hypothesis_request(tmp_path) + monkeypatch.setattr(module, "_hypothesis_service", lambda: service) + + assert module.record_hypothesis(request) == expected + assert service.requests == [request] + + +def test_record_hypothesis_accepts_mcp_json_mapping(tmp_path: Path) -> None: + request = _hypothesis_request(tmp_path) + + assert RecordHypothesisInput.model_validate(_json_request(request)) == request + + +def test_execute_candidate_passes_one_strict_object_to_the_service( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _module() + request = CandidateExecutionInput( + workspace_root=tmp_path / "accepted", + worktree_parent=tmp_path / "candidates", + benchmark_root=tmp_path / "benchmark", + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + experiment_id="experiment-one", + hypothesis_id=_HYPOTHESIS_ID, + ) + expected = CandidateExecutionObservation( + status=CandidateStatus.WARNING, + summary="The isolated candidate worktree is ready for the hypothesis edit.", + next_actions=("Edit the candidate, then poll.",), + artifacts=(str(tmp_path / "candidate"),), + phase=CandidatePhase.EDITING, + experiment_id="experiment-one", + hypothesis_id=_HYPOTHESIS_ID, + source_commit=_COMMIT, + worktree_path=tmp_path / "candidate", + outcome_receipts=(), + blockers=(), + ) + client = _FakeTraceClient() + store = _FakeOutcomeStore() + requests: list[CandidateExecutionInput] = [] + + def execute( + service: CandidateExecutionService, + candidate_request: CandidateExecutionInput, + ) -> CandidateExecutionObservation: + del service + requests.append(candidate_request) + return expected + + monkeypatch.setattr(module, "_client", lambda: client) + monkeypatch.setattr(module, "_outcome_store", lambda: store) + monkeypatch.setattr(CandidateExecutionService, "execute", execute) + + assert module.execute_candidate(request) == expected + assert requests == [request] + assert client.close_count == 1 + assert store.close_count == 1 + assert CandidateExecutionInput.model_validate(_json_request(request)) == request diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index 3b01148..b2bb8bb 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -23,15 +23,35 @@ class _McpManifest(BaseModel): def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: - path = Path(__file__).parents[1] / "plugins/openflywheel/.mcp.json" + root = Path(__file__).parents[1] + path = root / "plugins/openflywheel/.mcp.json" manifest = _McpManifest.model_validate_json(path.read_text(encoding="utf-8")) server = manifest.mcpServers["openflywheel"] - assert ( - "git+https://github.com/divo12/OpenFlyWheel.git@b4c785d9472ca1d6c245c41fa4d9985f0cef97fc" + "git+https://github.com/divo12/OpenFlyWheel.git@9041db3c08a89df0fe9f8f2476a303b46dd2812a" in server.args ) + runtime = (root / "src/ofw/mcp.py").read_text(encoding="utf-8") + assert "def record_hypothesis(" in runtime assert "openflywheel-mcp" in server.args assert "PLUGIN_ROOT" not in path.read_text(encoding="utf-8") assert "OPENFLYWHEEL_ROOT" not in path.read_text(encoding="utf-8") assert "LANGFUSE_SECRET_KEY" in server.env_vars + + +def test_plugin_and_runtime_package_versions_match() -> None: + root = Path(__file__).parents[1] + plugin_version_line = next( + line + for line in (root / "plugins/openflywheel/.codex-plugin/plugin.json") + .read_text(encoding="utf-8") + .splitlines() + if line.strip().startswith('"version":') + ) + package_version_line = next( + line + for line in (root / "pyproject.toml").read_text(encoding="utf-8").splitlines() + if line.startswith("version = ") + ) + + assert plugin_version_line.split('"')[3] == package_version_line.split('"')[1] diff --git a/tests/test_program_templates.py b/tests/test_program_templates.py index 0cd9cb3..8a6621a 100644 --- a/tests/test_program_templates.py +++ b/tests/test_program_templates.py @@ -60,6 +60,20 @@ def test_failure_pattern_miner_skill_is_packaged() -> None: assert "mine_failure_patterns" in skill.read_text(encoding="utf-8") +def test_program_routes_hypothesis_receipt_into_candidate_execution() -> None: + root = Path(__file__).parents[1] + skill = root / "plugins/openflywheel/skills/hypothesis-former/SKILL.md" + program = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") + + assert skill.is_file() + assert "record_hypothesis" in skill.read_text(encoding="utf-8") + assert "$hypothesis-former" in program + assert "stable hypothesis receipt" in program + assert "execute_candidate" in program + assert "returned candidate worktree" in program + assert "stop before admission" in program + + def test_base_program_stops_after_repeated_managed_mcp_timeout() -> None: content = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") content = " ".join(content.split()) diff --git a/tests/test_safe_file.py b/tests/test_safe_file.py new file mode 100644 index 0000000..1e87829 --- /dev/null +++ b/tests/test_safe_file.py @@ -0,0 +1,153 @@ +"""Shared descriptor-anchored immutable-file safety checks.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_child_directory, + open_directory, + publish_idempotent, + read_bounded, +) + + +def test_child_directory_swap_is_detected_without_redirecting_publication( + tmp_path: Path, +) -> None: + parent_path = tmp_path / "parent" + child_path = parent_path / "child" + moved_path = parent_path / "moved" + replacement_path = parent_path / "replacement" + child_path.mkdir(parents=True) + replacement_path.mkdir() + + with ( + pytest.raises(SafeFileFailure) as raised, + open_directory(parent_path) as parent, + open_child_directory(parent, "child", create=False) as child, + ): + child_path.rename(moved_path) + replacement_path.rename(child_path) + publish_idempotent( + child, + "artifact.json", + b"{}\n", + maximum_bytes=16, + subject="artifact", + ) + + assert raised.value.code is SafeFileErrorCode.DIRECTORY_CHANGED + assert not (child_path / "artifact.json").exists() + assert (moved_path / "artifact.json").read_bytes() == b"{}\n" + + +def test_regular_file_reader_rejects_device() -> None: + with open_directory(Path("/dev")) as directory, pytest.raises(SafeFileFailure) as raised: + read_bounded(directory, "null", maximum_bytes=16, subject="device") + + assert raised.value.code is SafeFileErrorCode.INVALID_FILE + + +def test_regular_file_reader_rejects_oversized_content(tmp_path: Path) -> None: + directory_path = tmp_path / "control" + directory_path.mkdir() + (directory_path / "policy.json").write_bytes(b"oversized") + + with open_directory(directory_path) as directory, pytest.raises(SafeFileFailure) as raised: + read_bounded(directory, "policy.json", maximum_bytes=4, subject="policy") + + assert raised.value.code is SafeFileErrorCode.TOO_LARGE + + +def test_directory_open_rejects_a_symlinked_ancestor(tmp_path: Path) -> None: + real_parent = tmp_path / "real" + directory = real_parent / "control" + directory.mkdir(parents=True) + linked_parent = tmp_path / "linked" + linked_parent.symlink_to(real_parent, target_is_directory=True) + + with pytest.raises(SafeFileFailure) as raised, open_directory(linked_parent / "control"): + pytest.fail("symlinked ancestor was followed") + + assert raised.value.code is SafeFileErrorCode.INVALID_FILE + + +def test_directory_swap_is_detected_without_redirecting_publication(tmp_path: Path) -> None: + directory_path = tmp_path / "control" + moved_path = tmp_path / "moved" + replacement_path = tmp_path / "replacement" + directory_path.mkdir() + replacement_path.mkdir() + + with pytest.raises(SafeFileFailure) as raised, open_directory(directory_path) as directory: + directory_path.rename(moved_path) + replacement_path.rename(directory_path) + publish_idempotent( + directory, + "policy.json", + b"{}\n", + maximum_bytes=16, + subject="policy", + ) + + assert raised.value.code is SafeFileErrorCode.DIRECTORY_CHANGED + assert not (directory_path / "policy.json").exists() + assert (moved_path / "policy.json").read_bytes() == b"{}\n" + + +def test_failed_atomic_link_leaves_no_publication_or_temporary_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + directory_path = tmp_path / "control" + directory_path.mkdir() + + def crash( + source: str, + destination: str, + *, + src_dir_fd: int, + dst_dir_fd: int, + follow_symlinks: bool, + ) -> None: + del source, destination, src_dir_fd, dst_dir_fd, follow_symlinks + raise OSError("simulated crash boundary") + + monkeypatch.setattr(os, "link", crash) + + with ( + open_directory(directory_path) as directory, + pytest.raises(OSError, match="simulated crash boundary"), + ): + publish_idempotent( + directory, + "policy.json", + b"{}\n", + maximum_bytes=16, + subject="policy", + ) + + assert tuple(directory_path.iterdir()) == () + + +def test_new_content_is_rejected_before_any_file_write_when_oversized(tmp_path: Path) -> None: + directory_path = tmp_path / "control" + directory_path.mkdir() + + with open_directory(directory_path) as directory, pytest.raises(SafeFileFailure) as raised: + publish_idempotent( + directory, + "policy.json", + b"too large", + maximum_bytes=4, + subject="policy", + ) + + assert raised.value.code is SafeFileErrorCode.TOO_LARGE + assert tuple(directory_path.iterdir()) == () diff --git a/tests/test_typing.py b/tests/test_typing.py index adc8fc3..ef8c7f1 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -35,7 +35,11 @@ def test_package_excludes_removed_harness_plane() -> None: def test_namespace_exports_authoritative_outcome_contract() -> None: + assert "EvaluatedRunBlocker" in package.__all__ + assert "EvaluatedRunReceipt" in package.__all__ + assert "EvaluatedTaskReceipt" in package.__all__ assert "OutcomeEvaluation" in package.__all__ + assert "RunSide" in package.__all__ assert "LangfuseOutcomeStore" in package.__all__ assert "EvidenceReference" in package.__all__ assert "TaskId" in package.__all__ @@ -83,3 +87,33 @@ def test_namespace_exports_workspace_preparation_contract() -> None: assert "PreparationStatus" in package.__all__ assert "PrepareWorkspaceInput" in package.__all__ assert "WorkspacePreparationObservation" in package.__all__ + + +def test_namespace_exports_policy_and_hypothesis_contracts_without_legacy_ownership() -> None: + expected = { + "ExperimentPolicySnapshot", + "FailurePatternReference", + "HarnessChangeTarget", + "HarnessHypothesis", + "HypothesisErrorCode", + "HypothesisId", + "HypothesisObservation", + "RecordHypothesisInput", + } + + assert expected <= set(package.__all__) + assert {"AssetAccess", "HarnessRevision", "HarnessRevisionId"}.isdisjoint(package.__all__) + + +def test_namespace_exports_candidate_contracts_without_restoring_runtime_plane() -> None: + expected = { + "CandidateExecutionInput", + "CandidateExecutionObservation", + "CandidateId", + "CandidatePhase", + "CandidateStatus", + } + + assert expected <= set(package.__all__) + assert {"E2BSandbox", "CanaryCase", "CommandLoop"}.isdisjoint(package.__all__) + assert {"CandidateBlocker", "CandidateOutcomeReceipt"}.isdisjoint(package.__all__) diff --git a/tests/test_workspace_preparation.py b/tests/test_workspace_preparation.py index 6116e98..4af4779 100644 --- a/tests/test_workspace_preparation.py +++ b/tests/test_workspace_preparation.py @@ -22,6 +22,12 @@ WorkspacePreparationService, ) from ofw.preparation.harbor import HarborBaselineRunner +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) from ofw.preparation.worktree import GitWorktreeGateway @@ -36,7 +42,13 @@ class _EnvironmentCapture(BaseModel): class _FailingRunner: def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: - return BaselineConfiguration(model="openai/gpt-5.4-mini", task_count=1) + return BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-1",), + benchmark_config_digest="sha256:" + "1" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ) def start(self, run: BaselineRun) -> int: raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "harbor") @@ -45,6 +57,24 @@ def summarize(self, run: BaselineRun) -> BaselineSummary | None: return None +class _AdoptingRunner: + def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: + return BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-pass", "task-fail"), + benchmark_config_digest="sha256:" + "1" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ) + + def start(self, run: BaselineRun) -> int: + raise AssertionError(f"adopted baseline must not launch Harbor: {run.job_path}") + + def summarize(self, run: BaselineRun) -> BaselineSummary | None: + assert run.job_path.name == "itsm-hermes-demo" + return BaselineSummary(2, 1, 1, 0, 0) + + def _git(root: Path, *arguments: str) -> str: return subprocess.run( ("git", "-C", str(root), *arguments), @@ -163,6 +193,7 @@ def _request( *, goal: str = "Reach full ITSM verifier pass rate.", expected_task_count: int = 2, + reuse_existing_baseline: bool = False, ) -> PrepareWorkspaceInput: return PrepareWorkspaceInput( experiment_id="itsm-hermes-demo", @@ -181,6 +212,7 @@ def _request( max_cost_per_task_usd=1.0, max_latency_seconds=600.0, max_baseline_seconds=60, + reuse_existing_baseline=reuse_existing_baseline, ) @@ -260,6 +292,15 @@ def test_prepare_workspace_creates_isolated_branch_commit_and_baseline( release=initialization_commit, session="itsm-hermes-demo", ) + policy = FileExperimentPolicyRepository().load(worktree, "itsm-hermes-demo") + assert policy.base_commit == ready.base_commit + assert policy.initialization_commit == ready.initialization_commit + assert policy.editable_paths == (Path("prompt.md"),) + assert policy.task_ids == ("tasks/task-pass", "tasks/task-fail") + assert policy.model == "openai/gpt-5.4-mini" + assert policy.concurrency == 1 + assert policy.max_retries == 0 + assert policy.controls_digest == policy.recomputed_controls_digest() assert (benchmark_root / "invocations.txt").read_text(encoding="utf-8") == "run\n" persisted_text = "\n".join( ( @@ -273,6 +314,10 @@ def test_prepare_workspace_creates_isolated_branch_commit_and_baseline( harness_root / ".git/ofw/preparations/itsm-hermes-demo/baseline.log" ).read_text(encoding="utf-8"), + ( + harness_root + / ".git/ofw/preparations/itsm-hermes-demo/policy.json" + ).read_text(encoding="utf-8"), ) ) assert "test-openai-key" not in persisted_text @@ -287,6 +332,46 @@ def test_prepare_workspace_creates_isolated_branch_commit_and_baseline( assert (benchmark_root / "invocations.txt").read_text(encoding="utf-8") == "run\n" +def test_prepare_workspace_adopts_existing_terminal_baseline_without_launching( + tmp_path: Path, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request( + harness_root, + worktree_parent, + benchmark_root, + harbor, + config, + reuse_existing_baseline=True, + ) + job = benchmark_root / "jobs/itsm-hermes-demo" + job.mkdir(parents=True) + (job / "result.json").write_text('{"finished_at":"2026-08-27T20:01:02Z"}', encoding="utf-8") + + service = WorkspacePreparationService( + runner=_AdoptingRunner(), + workspace=GitWorktreeGateway(), + base_program="# Base program\n", + itsm_program="## ITSM\n", + ) + + result = service.prepare(request) + + assert result.status is PreparationStatus.SUCCESS + assert result.phase is PreparationPhase.READY + assert result.terminal_trials == 2 + assert result.verifier_passes == 1 + assert result.verifier_failures == 1 + assert not (benchmark_root / "invocations.txt").exists() + policy = FileExperimentPolicyRepository().load(harness_root, request.experiment_id) + assert policy.task_ids == ("task-pass", "task-fail") + assert policy.baseline_reused is True + + def test_prepare_workspace_rejects_reused_id_with_different_configuration( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -338,6 +423,125 @@ def test_prepare_workspace_rejects_task_count_before_creating_branch(tmp_path: P assert result.error_code is PreparationErrorCode.TASK_COUNT_MISMATCH assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + +@pytest.mark.parametrize( + "content", + (b"x" * (2 * 1024 * 1024 + 1), b"\xff"), + ids=("oversized", "invalid-utf8"), +) +def test_prepare_workspace_sanitizes_invalid_harbor_config_reads( + tmp_path: Path, + content: bytes, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + config.write_bytes(content) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + +def test_prepare_workspace_rejects_oversized_model_before_creating_branch(tmp_path: Path) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + config.write_text( + config.read_text(encoding="utf-8").replace( + "openai/gpt-5.4-mini", + "m" * 257, + ), + encoding="utf-8", + ) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + +def test_policy_publication_failure_resumes_without_recreating_git_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + _credentials(monkeypatch) + original = FileExperimentPolicyRepository.publish + attempts = 0 + + def fail_once( + repository: FileExperimentPolicyRepository, + control_directory: Path, + policy: ExperimentPolicySnapshot, + ) -> Path: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_WRITE_FAILED, + request.experiment_id, + ) + return original(repository, control_directory, policy) + + monkeypatch.setattr(FileExperimentPolicyRepository, "publish", fail_once) + service = _service() + + first = service.prepare(request) + ready = _wait_until_ready(service, request) + + assert first.error_code is PreparationErrorCode.POLICY_WRITE_FAILED + assert ready.phase is PreparationPhase.READY + assert attempts == 2 + assert (benchmark_root / "invocations.txt").read_text(encoding="utf-8") == "run\n" + + +def test_policy_conflict_remains_typed_and_non_retryable_after_git_preparation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + _credentials(monkeypatch) + + def conflict( + repository: FileExperimentPolicyRepository, + control_directory: Path, + policy: ExperimentPolicySnapshot, + ) -> Path: + del repository, control_directory, policy + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_CONFLICT, + request.experiment_id, + ) + + monkeypatch.setattr(FileExperimentPolicyRepository, "publish", conflict) + service = _service() + + first = service.prepare(request) + repeated = service.prepare(request) + + assert first.error_code is repeated.error_code is PreparationErrorCode.POLICY_CONFLICT + assert first.retry is not None and "do not retry" in first.retry.lower() + assert first.stop_when is not None and "new experiment" in first.stop_when.lower() + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") + def test_prepare_workspace_input_rejects_relative_roots(tmp_path: Path) -> None: with pytest.raises(ValidationError): PrepareWorkspaceInput( @@ -358,6 +562,107 @@ def test_prepare_workspace_input_rejects_relative_roots(tmp_path: Path) -> None: ) +def test_prepare_workspace_input_rejects_nul_and_oversized_editable_paths(tmp_path: Path) -> None: + for editable in (Path("bad\x00path"), Path("p" * 1025)): + with pytest.raises(ValidationError): + PrepareWorkspaceInput( + experiment_id="demo", + harness_root=tmp_path, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=tmp_path, + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + expected_task_count=1, + editable_paths=(editable,), + goal="Improve.", + quality_target=1.0, + max_iterations=1, + no_improvement_limit=1, + max_baseline_seconds=60, + ) + + +def test_prepare_workspace_input_rejects_oversized_raw_path_before_normalization( + tmp_path: Path, +) -> None: + redundant = "./" * 600 + "prompt.md" + payload = f"""{{ + "experiment_id": "demo", + "harness_root": "{tmp_path}", + "base_ref": "HEAD", + "worktree_parent": "{tmp_path}", + "benchmark_root": "{tmp_path}", + "harbor_executable": "{tmp_path / 'harbor'}", + "harbor_config": "config.json", + "expected_task_count": 1, + "editable_paths": ["{redundant}"], + "goal": "Improve.", + "quality_target": 1.0, + "max_iterations": 1, + "no_improvement_limit": 1, + "max_baseline_seconds": 60 +}}""" + + with pytest.raises(ValidationError): + PrepareWorkspaceInput.model_validate_json(payload) + + +def test_prepare_workspace_input_rejects_oversized_direct_path(tmp_path: Path) -> None: + with pytest.raises(ValidationError): + PrepareWorkspaceInput( + experiment_id="demo", + harness_root=Path("/") / ("p" * 1025), + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=tmp_path, + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + expected_task_count=1, + editable_paths=(Path("prompt.md"),), + goal="Improve.", + quality_target=1.0, + max_iterations=1, + no_improvement_limit=1, + max_baseline_seconds=60, + ) + + +def test_legacy_preparation_state_requires_a_fresh_preparation_id(tmp_path: Path) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + control = GitWorktreeGateway().control_directory(harness_root, request.experiment_id) + control.mkdir(parents=True) + (control / "state.json").write_text('{"schema_version":1}\n', encoding="utf-8") + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED + assert result.retry is not None and "new experiment id" in result.retry.lower() + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + +def test_invalid_utf8_preparation_state_returns_typed_failure(tmp_path: Path) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + control = GitWorktreeGateway().control_directory(harness_root, request.experiment_id) + control.mkdir(parents=True) + (control / "state.json").write_bytes(b"\xff") + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.INVALID_BASELINE_RESULT + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + def test_prepare_workspace_input_accepts_json_path_strings(tmp_path: Path) -> None: config = PrepareWorkspaceInput.model_validate_json( f"""{{