From 8eead21e30e7ff536a63f086734d7a229dd02783 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:16:26 +0530 Subject: [PATCH 01/11] feat: add receipt-based promotion gate --- src/ofw/evolution/__init__.py | 5 + src/ofw/evolution/gate.py | 477 +++++++++++++++++++++++++++++ tests/test_gate.py | 556 ++++++++++++++++++++++++++++++++++ 3 files changed, 1038 insertions(+) create mode 100644 src/ofw/evolution/gate.py create mode 100644 tests/test_gate.py diff --git a/src/ofw/evolution/__init__.py b/src/ofw/evolution/__init__.py index 2d6146a..968d94f 100644 --- a/src/ofw/evolution/__init__.py +++ b/src/ofw/evolution/__init__.py @@ -13,6 +13,7 @@ from ofw.evolution.candidate_git import CandidateGitGateway from ofw.evolution.candidate_langfuse import LangfuseCandidateTraceLocator from ofw.evolution.candidate_service import CandidateExecutionService +from ofw.evolution.gate import PromotionDecision, PromotionReason, PromotionStatus, decide_promotion from ofw.evolution.hypothesis import ( FailurePatternReference, FailurePatternReferenceInput, @@ -40,6 +41,9 @@ "CandidateId", "CandidatePhase", "CandidateStatus", + "PromotionDecision", + "PromotionReason", + "PromotionStatus", "FailurePatternReference", "FailurePatternReferenceInput", "FileHypothesisRepository", @@ -54,4 +58,5 @@ "HypothesisStatus", "LangfuseCandidateTraceLocator", "RecordHypothesisInput", + "decide_promotion", ] diff --git a/src/ofw/evolution/gate.py b/src/ofw/evolution/gate.py new file mode 100644 index 0000000..368318e --- /dev/null +++ b/src/ofw/evolution/gate.py @@ -0,0 +1,477 @@ +"""Pure deterministic promotion gate over evaluated run receipts.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass +from enum import StrEnum + +from ofw.evaluation.outcome import ( + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + RunSide, + VerifierVerdict, +) +from ofw.evolution.candidate import candidate_policy_digest +from ofw.preparation.policy import ExperimentPolicySnapshot + + +class PromotionStatus(StrEnum): + ACCEPT = "accept" + ACCEPTED = "accept" + REJECT = "reject" + REJECTED = "reject" + INCONCLUSIVE = "inconclusive" + + +class PromotionReason(StrEnum): + IDENTITY_MISMATCH = "identity_mismatch" + POLICY_MISMATCH = "identity_mismatch" + CONTROLS_MISMATCH = "identity_mismatch" + GIT_IDENTITY_MISMATCH = "identity_mismatch" + RECEIPT_MISMATCH = "receipt_mismatch" + TASK_PARTITION_MISMATCH = "task_partition_mismatch" + TASK_MISMATCH = "task_partition_mismatch" + UNSUPPORTED_OUTCOME = "unsupported_outcome" + ERROR_OUTCOME = "error_outcome" + ABSTAIN_OUTCOME = "abstain_outcome" + UNVERIFIED_OUTCOME = "unverified_outcome" + MISSING_COST = "missing_cost" + MISSING_LATENCY = "missing_latency" + COST_LIMIT_EXCEEDED = "cost_limit_exceeded" + LATENCY_LIMIT_EXCEEDED = "latency_limit_exceeded" + PASS_REGRESSION = "pass_regression" + QUALITY_REGRESSION = "quality_regression" + NO_IMPROVEMENT = "no_improvement" + IMPROVEMENT = "improvement" + + +@dataclass(frozen=True, slots=True) +class PromotionDecision: + decision_id: str + policy_digest: str + accepted_run_id: str + candidate_run_id: str + status: PromotionStatus + reasons: tuple[PromotionReason, ...] + task_ids: tuple[str, ...] + accepted_passes: tuple[str, ...] + candidate_passes: tuple[str, ...] + accepted_quality: float + candidate_quality: float + accepted_cost_usd: float | None + candidate_cost_usd: float | None + accepted_latency_seconds: float | None + candidate_latency_seconds: float | None + canonical_json: str + + +def decide_promotion( + policy: ExperimentPolicySnapshot, + accepted_run: EvaluatedRunReceipt, + candidate_run: EvaluatedRunReceipt, +) -> PromotionDecision: + identity_reasons = _identity_reasons(policy, accepted_run, candidate_run) + partition_reasons = _partition_reasons(policy, accepted_run, candidate_run) + receipt_reasons = _receipt_reasons(policy, accepted_run, candidate_run) + reasons = _ordered_reasons(identity_reasons + partition_reasons + receipt_reasons) + if reasons: + return _decision(policy, accepted_run, candidate_run, PromotionStatus.INCONCLUSIVE, reasons) + + outcome_reasons = _outcome_reasons(accepted_run, candidate_run) + metric_reasons = _metric_reasons(policy, accepted_run, candidate_run) + reasons = _ordered_reasons(outcome_reasons + metric_reasons) + if reasons: + status = ( + PromotionStatus.REJECT + if _has_limit_violation(reasons) + else PromotionStatus.INCONCLUSIVE + ) + return _decision(policy, accepted_run, candidate_run, status, reasons) + + return _decide_quality(policy, accepted_run, candidate_run) + + +def _decide_quality( + policy: ExperimentPolicySnapshot, + accepted_run: EvaluatedRunReceipt, + candidate_run: EvaluatedRunReceipt, +) -> PromotionDecision: + + accepted_passes = _passes(accepted_run) + candidate_passes = _passes(candidate_run) + if not set(accepted_passes).issubset(candidate_passes): + return _decision( + policy, + accepted_run, + candidate_run, + PromotionStatus.REJECT, + (PromotionReason.PASS_REGRESSION,), + ) + if len(candidate_passes) <= len(accepted_passes): + return _decision( + policy, + accepted_run, + candidate_run, + PromotionStatus.REJECT, + (PromotionReason.NO_IMPROVEMENT,), + ) + return _decision( + policy, + accepted_run, + candidate_run, + PromotionStatus.ACCEPT, + (PromotionReason.IMPROVEMENT,), + ) + + +def _identity_reasons( + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> tuple[PromotionReason, ...]: + expected_policy = candidate_policy_digest(policy) + mismatch = _authority_identity_mismatch(expected_policy, policy, accepted, candidate) + if not mismatch: + mismatch = _run_identity_mismatch(accepted, candidate) + return (PromotionReason.IDENTITY_MISMATCH,) if mismatch else () + + +def _authority_identity_mismatch( + expected_policy: str, + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> bool: + return ( + _policy_identity_mismatch(expected_policy, accepted, candidate) + or _controls_identity_mismatch(policy, accepted, candidate) + or _side_identity_mismatch(accepted, candidate) + ) + + +def _run_identity_mismatch( + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> bool: + return _same_run(accepted, candidate) or _same_git_identity(accepted, candidate) + + +def _policy_identity_mismatch( + expected_policy: str, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> bool: + return accepted.policy_digest != expected_policy or candidate.policy_digest != expected_policy + + +def _controls_identity_mismatch( + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> bool: + return ( + accepted.controls_digest != policy.controls_digest + or candidate.controls_digest != policy.controls_digest + ) + + +def _side_identity_mismatch( + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> bool: + return accepted.side is not RunSide.ACCEPTED or candidate.side is not RunSide.CANDIDATE + + +def _same_run(accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt) -> bool: + return accepted.run_id == candidate.run_id + + +def _same_git_identity( + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> bool: + return ( + accepted.evaluated_commit == candidate.evaluated_commit + or accepted.evaluated_tree == candidate.evaluated_tree + ) + + +def _partition_reasons( + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> tuple[PromotionReason, ...]: + if accepted.task_ids != policy.task_ids or candidate.task_ids != policy.task_ids: + return (PromotionReason.TASK_PARTITION_MISMATCH,) + if not _partition_is_valid(accepted) or not _partition_is_valid(candidate): + return (PromotionReason.TASK_PARTITION_MISMATCH,) + return () + + +def _partition_is_valid(receipt: EvaluatedRunReceipt) -> bool: + task_ids = tuple(str(task_id) for task_id in receipt.task_ids) + result_ids = tuple(item.task_id for item in receipt.outcome_receipts) + tuple( + item.task_id for item in receipt.blockers + ) + return _has_exact_partition(result_ids, task_ids) and _is_ordered(result_ids, task_ids) + + +def _has_exact_partition(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> bool: + return ( + len(result_ids) == len(task_ids) + and len(set(result_ids)) == len(result_ids) + and set(result_ids) == set(task_ids) + ) + + +def _is_ordered(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> bool: + return result_ids == tuple(task_id for task_id in task_ids if task_id in result_ids) + + +def _receipt_reasons( + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> tuple[PromotionReason, ...]: + return _one_receipt_reasons(policy, accepted) + _one_receipt_reasons(policy, candidate) + + +def _one_receipt_reasons( + policy: ExperimentPolicySnapshot, + receipt: EvaluatedRunReceipt, +) -> tuple[PromotionReason, ...]: + if receipt.receipt_id != receipt.recomputed_id() or not _task_receipt_ids_unique(receipt): + return (PromotionReason.RECEIPT_MISMATCH,) + if any(not _task_receipt_is_valid(task, policy.verifier) for task in receipt.outcome_receipts): + return (PromotionReason.RECEIPT_MISMATCH,) + return () + + +def _task_receipt_ids_unique(receipt: EvaluatedRunReceipt) -> bool: + score_ids = tuple(item.score_id for item in receipt.outcome_receipts) + trace_ids = tuple(item.trace_id for item in receipt.outcome_receipts) + return len(score_ids) == len(set(score_ids)) and len(trace_ids) == len(set(trace_ids)) + + +def _task_receipt_is_valid(task: EvaluatedTaskReceipt, verifier: str) -> bool: + return ( + _verifier_matches(task.verifier_id, verifier) + and _score_is_valid(task) + and _metrics_are_valid(task) + ) + + +def _verifier_matches(verifier_id: str, verifier: str) -> bool: + return verifier_id == verifier or verifier_id.startswith(verifier + "@") + + +def _score_is_valid(task: EvaluatedTaskReceipt) -> bool: + expected = ( + 1.0 + if task.verdict is VerifierVerdict.PASS + else 0.0 + if task.verdict is VerifierVerdict.FAIL + else None + ) + return task.normalized_score == expected + + +def _metrics_are_valid(task: EvaluatedTaskReceipt) -> bool: + return ( + _metric_is_valid(task.normalized_score, 0.0, 1.0) + and _metric_is_valid(task.cost_usd, 0.0, 1_000_000.0) + and _metric_is_valid(task.latency_seconds, 0.0, 172800.0) + ) + + +def _metric_is_valid(value: float | None, minimum: float, maximum: float) -> bool: + return value is None or math.isfinite(value) and minimum <= value <= maximum + + +def _outcome_reasons( + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> tuple[PromotionReason, ...]: + return _run_outcome_reasons(accepted) + _run_outcome_reasons(candidate) + + +def _run_outcome_reasons(receipt: EvaluatedRunReceipt) -> tuple[PromotionReason, ...]: + reasons: list[PromotionReason] = [] + for blocker in receipt.blockers: + reasons.append(_blocker_reason(blocker.code)) + for task in receipt.outcome_receipts: + if task.verdict is VerifierVerdict.ERROR: + reasons.append(PromotionReason.ERROR_OUTCOME) + elif task.verdict is VerifierVerdict.ABSTAIN: + reasons.append(PromotionReason.ABSTAIN_OUTCOME) + return tuple(reasons) + + +def _blocker_reason(code: str) -> PromotionReason: + return ( + PromotionReason.UNSUPPORTED_OUTCOME + if "unsupported" in code + else PromotionReason.UNVERIFIED_OUTCOME + ) + + +def _metric_reasons( + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> tuple[PromotionReason, ...]: + reasons: list[PromotionReason] = [] + if policy.max_cost_per_task_usd is not None: + reason = _cost_reason(policy.max_cost_per_task_usd, accepted, candidate) + if reason is not None: + reasons.append(reason) + if policy.max_latency_seconds is not None: + reason = _latency_reason(policy.max_latency_seconds, accepted, candidate) + if reason is not None: + reasons.append(reason) + return tuple(reasons) + + +def _cost_reason( + limit: float, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> PromotionReason | None: + if _has_missing_cost(accepted) or _has_missing_cost(candidate): + return PromotionReason.MISSING_COST + if _exceeds_cost(candidate, limit): + return PromotionReason.COST_LIMIT_EXCEEDED + return None + + +def _latency_reason( + limit: float, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, +) -> PromotionReason | None: + if _has_missing_latency(accepted) or _has_missing_latency(candidate): + return PromotionReason.MISSING_LATENCY + if _exceeds_latency(candidate, limit): + return PromotionReason.LATENCY_LIMIT_EXCEEDED + return None + + +def _has_limit_violation(reasons: tuple[PromotionReason, ...]) -> bool: + return ( + PromotionReason.COST_LIMIT_EXCEEDED in reasons + or PromotionReason.LATENCY_LIMIT_EXCEEDED in reasons + ) + + +def _has_missing_cost(receipt: EvaluatedRunReceipt) -> bool: + return any(item.cost_usd is None for item in receipt.outcome_receipts) + + +def _has_missing_latency(receipt: EvaluatedRunReceipt) -> bool: + return any(item.latency_seconds is None for item in receipt.outcome_receipts) + + +def _exceeds_cost(receipt: EvaluatedRunReceipt, limit: float) -> bool: + return any( + item.cost_usd is not None and item.cost_usd > limit for item in receipt.outcome_receipts + ) + + +def _exceeds_latency(receipt: EvaluatedRunReceipt, limit: float) -> bool: + return any( + item.latency_seconds is not None and item.latency_seconds > limit + for item in receipt.outcome_receipts + ) + + +def _decision( + policy: ExperimentPolicySnapshot, + accepted: EvaluatedRunReceipt, + candidate: EvaluatedRunReceipt, + status: PromotionStatus, + reasons: tuple[PromotionReason, ...], +) -> PromotionDecision: + accepted_passes = _passes(accepted) + candidate_passes = _passes(candidate) + accepted_cost = _total_cost(accepted) + candidate_cost = _total_cost(candidate) + accepted_latency = _total_latency(accepted) + candidate_latency = _total_latency(candidate) + accepted_quality = _quality(accepted) + candidate_quality = _quality(candidate) + canonical_json = json.dumps( + ( + ("policy_digest", candidate_policy_digest(policy)), + ("controls_digest", policy.controls_digest), + ("accepted_receipt", accepted.model_dump_json()), + ("candidate_receipt", candidate.model_dump_json()), + ( + "metrics", + ( + ("accepted_quality", accepted_quality), + ("candidate_quality", candidate_quality), + ("accepted_cost_usd", accepted_cost), + ("candidate_cost_usd", candidate_cost), + ("accepted_latency_seconds", accepted_latency), + ("candidate_latency_seconds", candidate_latency), + ), + ), + ("status", status.value), + ("reasons", tuple(reason.value for reason in reasons)), + ), + separators=(",", ":"), + ) + decision_id = f"sha256:{hashlib.sha256(canonical_json.encode('utf-8')).hexdigest()}" + return PromotionDecision( + decision_id=decision_id, + policy_digest=candidate_policy_digest(policy), + accepted_run_id=accepted.run_id, + candidate_run_id=candidate.run_id, + status=status, + reasons=reasons, + task_ids=policy.task_ids, + accepted_passes=accepted_passes, + candidate_passes=candidate_passes, + accepted_quality=accepted_quality, + candidate_quality=candidate_quality, + accepted_cost_usd=accepted_cost, + candidate_cost_usd=candidate_cost, + accepted_latency_seconds=accepted_latency, + candidate_latency_seconds=candidate_latency, + canonical_json=canonical_json, + ) + + +def _passes(receipt: EvaluatedRunReceipt) -> tuple[str, ...]: + return tuple( + item.task_id for item in receipt.outcome_receipts if item.verdict is VerifierVerdict.PASS + ) + + +def _quality(receipt: EvaluatedRunReceipt) -> float: + return sum(item.normalized_score or 0.0 for item in receipt.outcome_receipts) / len( + receipt.task_ids + ) + + +def _total_cost(receipt: EvaluatedRunReceipt) -> float | None: + values = tuple(item.cost_usd for item in receipt.outcome_receipts) + return _total_metric(values) + + +def _total_latency(receipt: EvaluatedRunReceipt) -> float | None: + values = tuple(item.latency_seconds for item in receipt.outcome_receipts) + return _total_metric(values) + + +def _total_metric(values: tuple[float | None, ...]) -> float | None: + if any(value is None for value in values): + return None + return sum(value for value in values if value is not None) + + +def _ordered_reasons(reasons: tuple[PromotionReason, ...]) -> tuple[PromotionReason, ...]: + return tuple(reason for reason in PromotionReason if reason in reasons) diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..7b6c13d --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,556 @@ +import hashlib +import math +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest + +from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + RunSide, + VerifierVerdict, +) +from ofw.evolution.candidate import candidate_policy_digest +from ofw.evolution.gate import PromotionReason, PromotionStatus, decide_promotion +from ofw.preparation.contracts import ( + BaselineConfiguration, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import ExperimentPolicySnapshot, build_experiment_policy + +COMMIT = "a" * 40 +TREE = "b" * 40 + + +def _policy( + *, max_cost_per_task_usd: float | None = None, max_latency_seconds: float | None = None +) -> ExperimentPolicySnapshot: + request = PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=Path("/tmp/accepted"), + base_ref="HEAD", + worktree_parent=Path("/tmp/candidates"), + benchmark_root=Path("/tmp/benchmark"), + harbor_executable=Path("/tmp/harbor"), + harbor_config=Path("config.json"), + expected_task_count=3, + editable_paths=(Path("prompt.md"),), + goal="Improve verifier-backed quality.", + quality_target=1.0, + max_iterations=3, + no_improvement_limit=2, + max_cost_per_task_usd=max_cost_per_task_usd, + max_latency_seconds=max_latency_seconds, + max_baseline_seconds=600, + ) + return build_experiment_policy( + request, + PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=Path("/tmp/accepted"), + base_commit=COMMIT, + initialization_commit=COMMIT, + program_path=Path("/tmp/accepted/PROGRAM.md"), + ), + BaselineConfiguration( + model="model", + task_ids=("task-1", "task-2", "task-3"), + benchmark_config_digest="sha256:" + "c" * 64, + verifier="verifier", + environment="environment", + ), + ) + + +def _task( + task_id: str, + verdict: VerifierVerdict, + *, + cost_usd: float | None = 0.25, + latency_seconds: float | None = 1.5, + verifier_id: str = "verifier@checksum", +) -> EvaluatedTaskReceipt: + score = ( + 1.0 if verdict is VerifierVerdict.PASS else 0.0 if verdict is VerifierVerdict.FAIL else None + ) + return EvaluatedTaskReceipt( + task_id=task_id, + trace_id=f"trace-{task_id}", + score_id=f"score-{task_id}", + verdict=verdict, + verifier_id=verifier_id, + normalized_score=score, + cost_usd=cost_usd, + latency_seconds=latency_seconds, + ) + + +def _blocker(task_id: str, code: str = "unverified") -> EvaluatedRunBlocker: + return EvaluatedRunBlocker(task_id=task_id, code=code, subject="evidence") + + +def _receipt( + policy: ExperimentPolicySnapshot, + outcomes: tuple[EvaluatedTaskReceipt, ...], + blockers: tuple[EvaluatedRunBlocker, ...] = (), + *, + side: RunSide = RunSide.CANDIDATE, + run_id: str = "candidate-run", + evaluated_commit: str = COMMIT, + evaluated_tree: str = TREE, + policy_digest: str | None = None, + controls_digest: str | None = None, +) -> EvaluatedRunReceipt: + return EvaluatedRunReceipt.build( + run_id=run_id, + side=side, + policy_digest=policy_digest or candidate_policy_digest(policy), + controls_digest=controls_digest or policy.controls_digest, + evaluated_commit=evaluated_commit, + evaluated_tree=evaluated_tree, + task_ids=policy.task_ids, + outcome_receipts=outcomes, + blockers=blockers, + ) + + +def _runs( + policy: ExperimentPolicySnapshot, + accepted_verdicts: tuple[VerifierVerdict, ...], + candidate_verdicts: tuple[VerifierVerdict, ...], +) -> tuple[EvaluatedRunReceipt, EvaluatedRunReceipt]: + accepted = _receipt( + policy, + tuple( + _task(task_id, verdict) + for task_id, verdict in zip(policy.task_ids, accepted_verdicts, strict=True) + ), + side=RunSide.ACCEPTED, + run_id="accepted-run", + ) + candidate = _receipt( + policy, + tuple( + _task(task_id, verdict) + for task_id, verdict in zip(policy.task_ids, candidate_verdicts, strict=True) + ), + run_id="candidate-run", + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + return accepted, candidate + + +def _rehashed(receipt: EvaluatedRunReceipt) -> EvaluatedRunReceipt: + return EvaluatedRunReceipt.model_construct( + receipt_id=receipt.recomputed_id(), + run_id=receipt.run_id, + side=receipt.side, + policy_digest=receipt.policy_digest, + controls_digest=receipt.controls_digest, + evaluated_commit=receipt.evaluated_commit, + evaluated_tree=receipt.evaluated_tree, + task_ids=receipt.task_ids, + outcome_receipts=receipt.outcome_receipts, + blockers=receipt.blockers, + ) + + +def test_verdict_transition_matrix() -> None: + policy = _policy() + cases = ( + ( + (VerifierVerdict.PASS, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.PASS, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + PromotionStatus.REJECT, + PromotionReason.NO_IMPROVEMENT, + ), + ( + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.PASS, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + PromotionStatus.ACCEPT, + PromotionReason.IMPROVEMENT, + ), + ( + (VerifierVerdict.PASS, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + PromotionStatus.REJECT, + PromotionReason.PASS_REGRESSION, + ), + ( + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.ABSTAIN, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + PromotionStatus.INCONCLUSIVE, + PromotionReason.ABSTAIN_OUTCOME, + ), + ( + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.ERROR, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + PromotionStatus.INCONCLUSIVE, + PromotionReason.ERROR_OUTCOME, + ), + ) + for accepted_verdicts, candidate_verdicts, status, reason in cases: + accepted, candidate = _runs(policy, accepted_verdicts, candidate_verdicts) + decision = decide_promotion(policy, accepted, candidate) + assert decision.status is status + assert reason in decision.reasons + + +def test_unsupported_and_unverified_blockers_are_inconclusive() -> None: + policy = _policy() + accepted = _receipt( + policy, + tuple(_task(task_id, VerifierVerdict.FAIL) for task_id in policy.task_ids), + side=RunSide.ACCEPTED, + run_id="accepted-run", + ) + candidate = _receipt( + policy, + (_task("task-1", VerifierVerdict.FAIL), _task("task-2", VerifierVerdict.PASS)), + (_blocker("task-3", "unsupported_reward"),), + run_id="candidate-run", + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + decision = decide_promotion(policy, accepted, candidate) + assert decision.status is PromotionStatus.INCONCLUSIVE + assert decision.reasons == (PromotionReason.UNSUPPORTED_OUTCOME,) + all_blocked = _receipt( + policy, + (), + tuple(_blocker(task_id) for task_id in policy.task_ids), + run_id="candidate-run", + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + assert decide_promotion(policy, accepted, all_blocked).status is PromotionStatus.INCONCLUSIVE + + +@pytest.mark.parametrize( + "field", ("policy_digest", "controls_digest", "evaluated_commit", "evaluated_tree") +) +def test_identity_mismatch_is_inconclusive(field: str) -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + if field == "policy_digest": + candidate = _receipt( + policy, + candidate.outcome_receipts, + run_id=candidate.run_id, + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + policy_digest="sha256:" + "e" * 64, + ) + elif field == "controls_digest": + candidate = _receipt( + policy, + candidate.outcome_receipts, + run_id=candidate.run_id, + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + controls_digest="sha256:" + "e" * 64, + ) + elif field == "evaluated_commit": + candidate = _receipt( + policy, + candidate.outcome_receipts, + run_id=candidate.run_id, + evaluated_commit=COMMIT, + evaluated_tree="d" * 40, + ) + else: + candidate = _receipt( + policy, + candidate.outcome_receipts, + run_id=candidate.run_id, + evaluated_commit="c" * 40, + evaluated_tree=TREE, + ) + decision = decide_promotion(policy, accepted, candidate) + assert decision.status is PromotionStatus.INCONCLUSIVE + assert PromotionReason.IDENTITY_MISMATCH in decision.reasons + + +def test_wrong_sides_and_run_ids_are_inconclusive() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + wrong_side = _receipt( + policy, + candidate.outcome_receipts, + side=RunSide.ACCEPTED, + run_id=candidate.run_id, + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + same_run = _receipt( + policy, + candidate.outcome_receipts, + run_id=accepted.run_id, + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + assert ( + PromotionReason.IDENTITY_MISMATCH in decide_promotion(policy, accepted, wrong_side).reasons + ) + assert PromotionReason.IDENTITY_MISMATCH in decide_promotion(policy, accepted, same_run).reasons + + +@pytest.mark.parametrize( + "task_ids", + ( + ("task-1", "task-2"), + ("task-2", "task-1", "task-3"), + ("task-1", "task-2", "task-3", "task-4"), + ), +) +def test_task_set_must_match_policy_exactly(task_ids: tuple[str, ...]) -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + outcomes = tuple(_task(task_id, VerifierVerdict.FAIL) for task_id in task_ids) + tampered = EvaluatedRunReceipt.model_construct( + receipt_id=candidate.receipt_id, + run_id=candidate.run_id, + side=candidate.side, + policy_digest=candidate.policy_digest, + controls_digest=candidate.controls_digest, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + task_ids=task_ids, + outcome_receipts=outcomes, + blockers=candidate.blockers, + ) + decision = decide_promotion(policy, accepted, tampered) + assert decision.status is PromotionStatus.INCONCLUSIVE + assert PromotionReason.TASK_PARTITION_MISMATCH in decision.reasons + + +def test_duplicate_receipt_id_and_tampered_receipt_hash_are_inconclusive() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + duplicate = EvaluatedTaskReceipt.model_construct( + task_id=candidate.outcome_receipts[0].task_id, + trace_id=candidate.outcome_receipts[0].trace_id, + score_id=candidate.outcome_receipts[1].score_id, + verdict=candidate.outcome_receipts[0].verdict, + verifier_id=candidate.outcome_receipts[0].verifier_id, + normalized_score=candidate.outcome_receipts[0].normalized_score, + cost_usd=candidate.outcome_receipts[0].cost_usd, + latency_seconds=candidate.outcome_receipts[0].latency_seconds, + ) + tampered = EvaluatedRunReceipt.model_construct( + receipt_id=candidate.receipt_id, + run_id=candidate.run_id, + side=candidate.side, + policy_digest=candidate.policy_digest, + controls_digest=candidate.controls_digest, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + task_ids=candidate.task_ids, + outcome_receipts=(duplicate,) + candidate.outcome_receipts[1:], + blockers=candidate.blockers, + ) + assert PromotionReason.RECEIPT_MISMATCH in decide_promotion(policy, accepted, tampered).reasons + duplicate_partition = EvaluatedRunReceipt.model_construct( + receipt_id=candidate.receipt_id, + run_id=candidate.run_id, + side=candidate.side, + policy_digest=candidate.policy_digest, + controls_digest=candidate.controls_digest, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + task_ids=candidate.task_ids, + outcome_receipts=( + candidate.outcome_receipts[0], + candidate.outcome_receipts[0], + candidate.outcome_receipts[2], + ), + blockers=(), + ) + assert ( + PromotionReason.TASK_PARTITION_MISMATCH + in decide_promotion(policy, accepted, duplicate_partition).reasons + ) + + +def test_receipts_must_name_the_configured_verifier() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + invalid = _receipt( + policy, + ( + _task("task-1", VerifierVerdict.FAIL, verifier_id="other-verifier"), + candidate.outcome_receipts[1], + candidate.outcome_receipts[2], + ), + run_id=candidate.run_id, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + ) + assert PromotionReason.RECEIPT_MISMATCH in decide_promotion(policy, accepted, invalid).reasons + + +@pytest.mark.parametrize( + ("cost", "latency", "reason"), + ((None, 1.5, PromotionReason.MISSING_COST), (0.25, None, PromotionReason.MISSING_LATENCY)), +) +def test_configured_metrics_require_explicit_measurements( + cost: float | None, latency: float | None, reason: PromotionReason +) -> None: + policy = _policy( + max_cost_per_task_usd=1.0 if cost is None else None, + max_latency_seconds=1.0 if latency is None else None, + ) + accepted = _receipt( + policy, + ( + _task("task-1", VerifierVerdict.FAIL, cost_usd=cost, latency_seconds=latency), + _task("task-2", VerifierVerdict.FAIL, cost_usd=cost, latency_seconds=latency), + _task("task-3", VerifierVerdict.FAIL, cost_usd=cost, latency_seconds=latency), + ), + side=RunSide.ACCEPTED, + run_id="accepted-run", + ) + candidate = _receipt( + policy, + ( + _task("task-1", VerifierVerdict.FAIL, cost_usd=cost, latency_seconds=latency), + _task("task-2", VerifierVerdict.PASS, cost_usd=cost, latency_seconds=latency), + _task("task-3", VerifierVerdict.FAIL, cost_usd=cost, latency_seconds=latency), + ), + run_id="candidate-run", + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + decision = decide_promotion(policy, accepted, candidate) + assert decision.status is PromotionStatus.INCONCLUSIVE + assert reason in decision.reasons + + +@pytest.mark.parametrize( + ("field", "limit", "reason"), + ( + ("cost_usd", 1.0, PromotionReason.COST_LIMIT_EXCEEDED), + ("latency_seconds", 1.0, PromotionReason.LATENCY_LIMIT_EXCEEDED), + ), +) +def test_candidate_metric_limits_are_inclusive_and_excess_rejects( + field: str, limit: float, reason: PromotionReason +) -> None: + policy = _policy( + max_cost_per_task_usd=limit if field == "cost_usd" else None, + max_latency_seconds=limit if field == "latency_seconds" else None, + ) + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + candidate_verdicts = (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL) + boundary = tuple( + _task( + task_id, + verdict, + cost_usd=limit if field == "cost_usd" else 0.25, + latency_seconds=limit if field == "latency_seconds" else 1.5, + ) + for task_id, verdict in zip(policy.task_ids, candidate_verdicts, strict=True) + ) + boundary_candidate = _receipt( + policy, boundary, run_id="candidate-run", evaluated_commit="c" * 40, evaluated_tree="d" * 40 + ) + assert decide_promotion(policy, accepted, boundary_candidate).status is PromotionStatus.ACCEPT + excess = tuple( + _task( + task_id, + verdict, + cost_usd=limit + 0.01 if field == "cost_usd" else 0.25, + latency_seconds=limit + 0.01 if field == "latency_seconds" else 1.5, + ) + for task_id, verdict in zip(policy.task_ids, candidate_verdicts, strict=True) + ) + excess_candidate = _receipt( + policy, excess, run_id="candidate-run", evaluated_commit="c" * 40, evaluated_tree="d" * 40 + ) + decision = decide_promotion(policy, accepted, excess_candidate) + assert decision.status is PromotionStatus.REJECT + assert reason in decision.reasons + + +def test_non_finite_tampered_metric_is_inconclusive() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + task = EvaluatedTaskReceipt.model_construct( + task_id=candidate.outcome_receipts[1].task_id, + trace_id=candidate.outcome_receipts[1].trace_id, + score_id=candidate.outcome_receipts[1].score_id, + verdict=candidate.outcome_receipts[1].verdict, + verifier_id=candidate.outcome_receipts[1].verifier_id, + normalized_score=math.nan, + cost_usd=candidate.outcome_receipts[1].cost_usd, + latency_seconds=candidate.outcome_receipts[1].latency_seconds, + ) + tampered = EvaluatedRunReceipt.model_construct( + receipt_id=candidate.receipt_id, + run_id=candidate.run_id, + side=candidate.side, + policy_digest=candidate.policy_digest, + controls_digest=candidate.controls_digest, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + task_ids=candidate.task_ids, + outcome_receipts=(candidate.outcome_receipts[0], task, candidate.outcome_receipts[2]), + blockers=candidate.blockers, + ) + tampered = _rehashed(tampered) + decision = decide_promotion(policy, accepted, tampered) + assert decision.status is PromotionStatus.INCONCLUSIVE + assert PromotionReason.RECEIPT_MISMATCH in decision.reasons + + +def test_decision_is_immutable_deterministic_and_golden() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + first = decide_promotion(policy, accepted, candidate) + second = decide_promotion(policy, accepted, candidate) + assert first == second + assert ( + first.decision_id == "sha256:" + hashlib.sha256(first.canonical_json.encode()).hexdigest() + ) + assert ( + first.decision_id + == "sha256:e6427af38ba089471ff626f88b6d84a9ba6592ed274a717ea2dbc375fbc0dd41" + ) + with pytest.raises(FrozenInstanceError): + first.status = PromotionStatus.ACCEPT # type: ignore[misc] From 63a5048ab1401da8a5e08c3fbf6709b6b129410b Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:54:03 +0530 Subject: [PATCH 02/11] fix: harden promotion gate evidence handling --- src/ofw/evolution/gate.py | 30 +++++++++---- tests/test_gate.py | 95 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src/ofw/evolution/gate.py b/src/ofw/evolution/gate.py index 368318e..e28d3bf 100644 --- a/src/ofw/evolution/gate.py +++ b/src/ofw/evolution/gate.py @@ -212,11 +212,21 @@ def _partition_reasons( def _partition_is_valid(receipt: EvaluatedRunReceipt) -> bool: - task_ids = tuple(str(task_id) for task_id in receipt.task_ids) - result_ids = tuple(item.task_id for item in receipt.outcome_receipts) + tuple( - item.task_id for item in receipt.blockers + task_ids, outcome_ids, blocker_ids = _partition_values(receipt) + result_ids = outcome_ids + blocker_ids + if not _has_exact_partition(result_ids, task_ids): + return False + return _is_ordered(outcome_ids, task_ids) and _is_ordered(blocker_ids, task_ids) + + +def _partition_values( + receipt: EvaluatedRunReceipt, +) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + return ( + tuple(str(task_id) for task_id in receipt.task_ids), + tuple(item.task_id for item in receipt.outcome_receipts), + tuple(item.task_id for item in receipt.blockers), ) - return _has_exact_partition(result_ids, task_ids) and _is_ordered(result_ids, task_ids) def _has_exact_partition(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> bool: @@ -340,10 +350,10 @@ def _cost_reason( accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt, ) -> PromotionReason | None: - if _has_missing_cost(accepted) or _has_missing_cost(candidate): - return PromotionReason.MISSING_COST if _exceeds_cost(candidate, limit): return PromotionReason.COST_LIMIT_EXCEEDED + if _has_missing_cost(accepted) or _has_missing_cost(candidate): + return PromotionReason.MISSING_COST return None @@ -352,10 +362,10 @@ def _latency_reason( accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt, ) -> PromotionReason | None: - if _has_missing_latency(accepted) or _has_missing_latency(candidate): - return PromotionReason.MISSING_LATENCY if _exceeds_latency(candidate, limit): return PromotionReason.LATENCY_LIMIT_EXCEEDED + if _has_missing_latency(accepted) or _has_missing_latency(candidate): + return PromotionReason.MISSING_LATENCY return None @@ -458,11 +468,15 @@ def _quality(receipt: EvaluatedRunReceipt) -> float: def _total_cost(receipt: EvaluatedRunReceipt) -> float | None: + if receipt.blockers: + return None values = tuple(item.cost_usd for item in receipt.outcome_receipts) return _total_metric(values) def _total_latency(receipt: EvaluatedRunReceipt) -> float | None: + if receipt.blockers: + return None values = tuple(item.latency_seconds for item in receipt.outcome_receipts) return _total_metric(values) diff --git a/tests/test_gate.py b/tests/test_gate.py index 7b6c13d..28ad633 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -230,6 +230,30 @@ def test_unsupported_and_unverified_blockers_are_inconclusive() -> None: assert decide_promotion(policy, accepted, all_blocked).status is PromotionStatus.INCONCLUSIVE +def test_interleaved_outcomes_and_blockers_follow_each_partition_order() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + interleaved = EvaluatedRunReceipt.model_construct( + receipt_id=candidate.receipt_id, + run_id=candidate.run_id, + side=candidate.side, + policy_digest=candidate.policy_digest, + controls_digest=candidate.controls_digest, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + task_ids=candidate.task_ids, + outcome_receipts=(candidate.outcome_receipts[0], candidate.outcome_receipts[2]), + blockers=(_blocker("task-2"),), + ) + decision = decide_promotion(policy, accepted, _rehashed(interleaved)) + assert decision.status is PromotionStatus.INCONCLUSIVE + assert decision.reasons == (PromotionReason.UNVERIFIED_OUTCOME,) + + @pytest.mark.parametrize( "field", ("policy_digest", "controls_digest", "evaluated_commit", "evaluated_tree") ) @@ -450,6 +474,77 @@ def test_configured_metrics_require_explicit_measurements( assert reason in decision.reasons +@pytest.mark.parametrize( + ("field", "limit", "reason"), + ( + ("cost_usd", 1.0, PromotionReason.COST_LIMIT_EXCEEDED), + ("latency_seconds", 1.0, PromotionReason.LATENCY_LIMIT_EXCEEDED), + ), +) +def test_candidate_metric_violation_wins_over_missing_accepted_metric( + field: str, limit: float, reason: PromotionReason +) -> None: + policy = _policy( + max_cost_per_task_usd=limit if field == "cost_usd" else None, + max_latency_seconds=limit if field == "latency_seconds" else None, + ) + accepted = _receipt( + policy, + tuple( + _task(task_id, VerifierVerdict.FAIL, cost_usd=None, latency_seconds=None) + for task_id in policy.task_ids + ), + side=RunSide.ACCEPTED, + run_id="accepted-run", + ) + candidate = _receipt( + policy, + tuple( + _task( + task_id, + verdict, + cost_usd=limit + 0.01 if field == "cost_usd" else 0.25, + latency_seconds=limit + 0.01 if field == "latency_seconds" else 1.5, + ) + for task_id, verdict in zip( + policy.task_ids, + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + strict=True, + ) + ), + run_id="candidate-run", + evaluated_commit="c" * 40, + evaluated_tree="d" * 40, + ) + decision = decide_promotion(policy, accepted, candidate) + assert decision.status is PromotionStatus.REJECT + assert reason in decision.reasons + + +def test_incomplete_run_metrics_are_not_reported_as_totals() -> None: + policy = _policy() + accepted, candidate = _runs( + policy, + (VerifierVerdict.FAIL, VerifierVerdict.FAIL, VerifierVerdict.FAIL), + (VerifierVerdict.FAIL, VerifierVerdict.PASS, VerifierVerdict.FAIL), + ) + incomplete = EvaluatedRunReceipt.model_construct( + receipt_id=candidate.receipt_id, + run_id=candidate.run_id, + side=candidate.side, + policy_digest=candidate.policy_digest, + controls_digest=candidate.controls_digest, + evaluated_commit=candidate.evaluated_commit, + evaluated_tree=candidate.evaluated_tree, + task_ids=candidate.task_ids, + outcome_receipts=(candidate.outcome_receipts[0],), + blockers=(_blocker("task-2"), _blocker("task-3")), + ) + decision = decide_promotion(policy, accepted, _rehashed(incomplete)) + assert decision.candidate_cost_usd is None + assert decision.candidate_latency_seconds is None + + @pytest.mark.parametrize( ("field", "limit", "reason"), ( From bd36934cb12a7a62d43c32aea87e6e527a8fb476 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:59:25 +0530 Subject: [PATCH 03/11] test: refresh gate golden digest --- tests/test_gate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gate.py b/tests/test_gate.py index 28ad633..6f7fc9d 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -645,7 +645,7 @@ def test_decision_is_immutable_deterministic_and_golden() -> None: ) assert ( first.decision_id - == "sha256:e6427af38ba089471ff626f88b6d84a9ba6592ed274a717ea2dbc375fbc0dd41" + == "sha256:e069074fc405ed919291bc16626a3c82865d8bb9a43bd5b8468ee1858529792f" ) with pytest.raises(FrozenInstanceError): first.status = PromotionStatus.ACCEPT # type: ignore[misc] From 31b53b70dc435696a92ad0422b177c60521bb9b4 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 16:45:20 +0530 Subject: [PATCH 04/11] feat: add evolution controller and append-only ledger --- .../openflywheel/program_templates/base.md | 56 +- src/ofw/__init__.py | 33 + src/ofw/evolution/__init__.py | 73 +- src/ofw/evolution/controller.py | 1055 +++++++++++++++++ src/ofw/evolution/ledger.py | 655 ++++++++++ src/ofw/mcp.py | 29 +- src/ofw/preparation/__init__.py | 2 + src/ofw/preparation/policy.py | 5 + src/ofw/preparation/templates/base.md | 56 +- tests/__init__.py | 0 tests/support_policy.py | 67 ++ tests/test_evolution_controller.py | 425 +++++++ tests/test_evolution_ledger.py | 196 +++ tests/test_openflywheel_mcp.py | 4 + 14 files changed, 2586 insertions(+), 70 deletions(-) create mode 100644 src/ofw/evolution/controller.py create mode 100644 src/ofw/evolution/ledger.py create mode 100644 tests/__init__.py create mode 100644 tests/support_policy.py create mode 100644 tests/test_evolution_controller.py create mode 100644 tests/test_evolution_ledger.py diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index c440f02..eb2f5d2 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -4,8 +4,8 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Record one evidence-backed hypothesis, execute its isolated candidate under the canonical -experiment policy, then stop before admission. +Call `evolution_status`, then `advance_evolution` for exactly one next action under the +canonical experiment policy. Stop before admission. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an @@ -31,38 +31,26 @@ Target only exact paths allowed by the canonical experiment policy. Edit them on candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks, verifier, model, reasoning budget, observability identity, or this program. -Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control -flow changes unless the evidence requires the combination. - -## Optimization loop - -### 2. Analyze failures - -Start from verifier-backed failed outcomes. Use bounded trace queries to locate relevant -evidence, then inspect only the spans needed to explain the observed behavior. Do not load -or copy complete traces when filters answer the question, and do not inspect held-out -trajectory content. - -### 3. Form one hypothesis - -State the failure pattern, supporting trace and verifier evidence, proposed harness change, -expected improvement, and possible regressions. Stop if the evidence cannot distinguish -between materially different changes. - -Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported -pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call -`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution. - -### 4. Execute one candidate - -Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling -candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated -worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the -returned candidate worktree, then call `execute_candidate` again with the identical request. - -Poll identical requests while the candidate is running. Retain its candidate ID, Git commit, -trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally, -change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree. +Keep one focused hypothesis per iteration. The controller owns phase, budgets, stop reasons, +and ledger truth; never append events or perform generic transitions yourself. + +## Controller loop + +1. Use bounded trace queries and `$failure-miner`, `record_failure`, `$failure-pattern-miner`, + `mine_failure_patterns`, `$failure-curator`, and `record_failure_curation` to retain only + typed evidence; do not copy Langfuse trace payloads. +2. Use `$hypothesis-former` and `record_hypothesis`, then pass the stable hypothesis receipt to + `advance_evolution`. +3. When the controller returns a candidate worktree, edit only its declared targets and use the + returned candidate worktree with `execute_candidate`; retain the candidate and evaluated run receipts. +4. Pass the existing `PromotionDecision` to `advance_evolution`. Accepted candidates remain + `AwaitingPublication` until PR5; do not publish, merge, push, or install. + +The controller returns the required next action. Use bounded trace queries and the failure +skills to retain only typed evidence; do not copy Langfuse trace payloads. Form one hypothesis +with `$hypothesis-former` and `record_hypothesis`, then pass its receipt to `advance_evolution`. +Call `execute_candidate` when requested, edit only its declared targets, and repeat the identical +request. Pass the existing `PromotionDecision` to `advance_evolution`. ## Package boundary diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 7287b86..c36094b 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -49,6 +49,8 @@ VerifierVerdict, ) from ofw.evolution import ( + AdvanceEvolutionInput, + CandidateBlocker, CandidateBlockerCode, CandidateErrorCode, CandidateExecutionInput, @@ -57,8 +59,23 @@ CandidateId, CandidatePhase, CandidateStatus, + EvolutionAdvanceAction, + EvolutionController, + EvolutionControllerErrorCode, + EvolutionControllerFailure, + EvolutionEvent, + EvolutionEventDraft, + EvolutionEventPage, + EvolutionEventType, + EvolutionLedgerErrorCode, + EvolutionLedgerFailure, + EvolutionObservation, + EvolutionPhase, + EvolutionStatus, + EvolutionStopReason, FailurePatternReference, FailurePatternReferenceInput, + FileEvolutionLedger, HarnessChangeTarget, HarnessChangeTargetInput, HarnessHypothesis, @@ -96,6 +113,22 @@ "CandidateId", "CandidatePhase", "CandidateStatus", + "AdvanceEvolutionInput", + "EvolutionAdvanceAction", + "EvolutionController", + "EvolutionControllerErrorCode", + "EvolutionControllerFailure", + "EvolutionObservation", + "EvolutionPhase", + "EvolutionStatus", + "EvolutionStopReason", + "EvolutionEvent", + "EvolutionEventDraft", + "EvolutionEventPage", + "EvolutionEventType", + "EvolutionLedgerErrorCode", + "EvolutionLedgerFailure", + "FileEvolutionLedger", "CollectionError", "CollectionErrorCode", "ComponentKind", diff --git a/src/ofw/evolution/__init__.py b/src/ofw/evolution/__init__.py index 968d94f..99478ba 100644 --- a/src/ofw/evolution/__init__.py +++ b/src/ofw/evolution/__init__.py @@ -13,7 +13,23 @@ from ofw.evolution.candidate_git import CandidateGitGateway from ofw.evolution.candidate_langfuse import LangfuseCandidateTraceLocator from ofw.evolution.candidate_service import CandidateExecutionService -from ofw.evolution.gate import PromotionDecision, PromotionReason, PromotionStatus, decide_promotion +from ofw.evolution.controller import ( + AdvanceEvolutionInput, + EvolutionAdvanceAction, + EvolutionController, + EvolutionControllerErrorCode, + EvolutionControllerFailure, + EvolutionObservation, + EvolutionPhase, + EvolutionStatus, + EvolutionStopReason, +) +from ofw.evolution.gate import ( + PromotionDecision, + PromotionReason, + PromotionStatus, + decide_promotion, +) from ofw.evolution.hypothesis import ( FailurePatternReference, FailurePatternReferenceInput, @@ -29,6 +45,30 @@ RecordHypothesisInput, ) from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.evolution.ledger import ( + CandidateAccepted, + CandidatePrepared, + CandidateRejected, + CandidateSubmitted, + EvolutionEvent, + EvolutionEventDraft, + EvolutionEventPage, + EvolutionEventPayload, + EvolutionEventType, + EvolutionLedgerErrorCode, + EvolutionLedgerFailure, + EvolutionStarted, + ExternalOperation, + ExternalOperationBlocked, + ExternalOperationIntent, + FileEvolutionLedger, + GateDecided, + HypothesisLinked, + ReleasePublished, + ReleaseRolledBack, + RunCompleted, + RunStarted, +) __all__ = [ "CandidateBlockerCode", @@ -41,6 +81,37 @@ "CandidateId", "CandidatePhase", "CandidateStatus", + "AdvanceEvolutionInput", + "EvolutionAdvanceAction", + "EvolutionController", + "EvolutionControllerErrorCode", + "EvolutionControllerFailure", + "EvolutionObservation", + "EvolutionPhase", + "EvolutionStatus", + "EvolutionStopReason", + "EvolutionEvent", + "EvolutionEventDraft", + "EvolutionEventPayload", + "EvolutionEventPage", + "EvolutionEventType", + "EvolutionStarted", + "HypothesisLinked", + "CandidatePrepared", + "CandidateSubmitted", + "RunStarted", + "RunCompleted", + "GateDecided", + "CandidateAccepted", + "CandidateRejected", + "ReleasePublished", + "ReleaseRolledBack", + "ExternalOperation", + "ExternalOperationIntent", + "ExternalOperationBlocked", + "EvolutionLedgerErrorCode", + "EvolutionLedgerFailure", + "FileEvolutionLedger", "PromotionDecision", "PromotionReason", "PromotionStatus", diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py new file mode 100644 index 0000000..e10a8c7 --- /dev/null +++ b/src/ofw/evolution/controller.py @@ -0,0 +1,1055 @@ +"""Resumable one-step evolution controller over the typed event ledger.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import StrEnum +from pathlib import Path +from typing import Protocol + +from pydantic import Field, model_validator + +from ofw.evaluation.outcome import EvaluatedRunReceipt, RunSide +from ofw.evolution.candidate import candidate_policy_digest +from ofw.evolution.gate import PromotionDecision, PromotionStatus +from ofw.evolution.hypothesis import HarnessHypothesis, HypothesisFailure +from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.evolution.ledger import ( + CandidateAccepted, + CandidatePrepared, + CandidateRejected, + CandidateSubmitted, + EvolutionEvent, + EvolutionEventDraft, + EvolutionEventPayload, + EvolutionEventType, + EvolutionLedgerFailure, + EvolutionStarted, + EvolutionStopped, + EvolutionStopReason, + ExternalOperation, + ExternalOperationBlocked, + ExternalOperationIntent, + FileEvolutionLedger, + GateDecided, + HypothesisLinked, + RunCompleted, + RunStarted, +) +from ofw.preparation.contracts import StrictModel +from ofw.preparation.policy import ( + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) + +_DIGEST = r"sha256:[0-9a-f]{64}" +_IDENTIFIER = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" + +__all__ = [ + "AdvanceEvolutionInput", + "EvolutionAdvanceAction", + "EvolutionController", + "EvolutionControllerErrorCode", + "EvolutionControllerFailure", + "EvolutionObservation", + "EvolutionPhase", + "EvolutionStatus", + "EvolutionStopReason", +] + + +class EvolutionPhase(StrEnum): + AWAITING_HYPOTHESIS = "awaiting_hypothesis" + AWAITING_CANDIDATE = "awaiting_candidate" + CANDIDATE_RUNNING = "candidate_running" + GATE_READY = "gate_ready" + AWAITING_PUBLICATION = "awaiting_publication" + BLOCKED = "blocked" + STOPPED = "stopped" + + +class EvolutionAdvanceAction(StrEnum): + AUTO = "auto" + LINK_HYPOTHESIS = "link_hypothesis" + PREPARE_CANDIDATE = "prepare_candidate" + SUBMIT_CANDIDATE = "submit_candidate" + COMPLETE_RUN = "complete_run" + DECIDE_GATE = "decide_gate" + RETRY = "retry" + BLOCK = "block" + STOP = "stop" + PUBLISH = "publish" + + +class EvolutionStatus(StrEnum): + SUCCESS = "success" + WARNING = "warning" + ERROR = "error" + + +class EvolutionControllerErrorCode(StrEnum): + POLICY_INVALID = "policy_invalid" + LEDGER_INVALID = "ledger_invalid" + REQUEST_CONFLICT = "request_conflict" + INVALID_TRANSITION = "invalid_transition" + MISSING_INPUT = "missing_input" + STALE_RECEIPT = "stale_receipt" + PUBLICATION_REQUIRED = "publication_required" + STOPPED = "stopped" + EVIDENCE_UNAVAILABLE = "evidence_unavailable" + MAX_ITERATIONS = "max_iterations" + NO_IMPROVEMENT = "no_improvement" + + +class EvolutionControllerFailure(Exception): + __slots__ = ("code", "subject") + + def __init__(self, code: EvolutionControllerErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +class AdvanceEvolutionInput(StrictModel): + workspace_root: Path + experiment_id: str = Field( + min_length=1, max_length=80, pattern=r"[a-z0-9]+(?:-[a-z0-9]+)*" + ) + request_id: str = Field(min_length=1, max_length=256, pattern=_IDENTIFIER) + action: EvolutionAdvanceAction = EvolutionAdvanceAction.AUTO + hypothesis_id: str | None = Field(default=None, pattern=_DIGEST) + source_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_workspace_id: str | None = Field( + default=None, max_length=256, pattern=_IDENTIFIER + ) + candidate_id: str | None = Field(default=None, pattern=_DIGEST) + candidate_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + run_id: str | None = Field(default=None, max_length=256, pattern=_IDENTIFIER) + candidate_receipt_id: str | None = Field(default=None, pattern=_DIGEST) + evaluated_run_receipt: EvaluatedRunReceipt | None = None + promotion_decision: PromotionDecision | None = None + release_id: str | None = Field(default=None, max_length=256, pattern=_IDENTIFIER) + stop_reason: EvolutionStopReason | None = None + blocker_reason: str | None = Field( + default=None, min_length=1, max_length=256, pattern=_IDENTIFIER + ) + baseline_deadline_exceeded: bool = False + evidence_available: bool = True + requested_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + @model_validator(mode="after") + def validate_root(self) -> AdvanceEvolutionInput: + if not self.workspace_root.is_absolute(): + raise ValueError("workspace_root must be absolute") + if self.action is EvolutionAdvanceAction.STOP and self.stop_reason is None: + raise ValueError("stop_reason is required") + return self + + def digest(self) -> str: + content = self.model_dump_json(exclude={"requested_at"}) + return "sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest() + + +class EvolutionObservation(StrictModel): + experiment_id: str = Field(min_length=1, max_length=80) + status: EvolutionStatus + phase: EvolutionPhase + summary: str = Field(min_length=1, max_length=256) + next_actions: tuple[str, ...] = Field(max_length=3) + sequence: int = Field(strict=True, ge=0) + iteration: int = Field(strict=True, ge=0, le=100) + hypothesis_id: str | None = Field(default=None, pattern=_DIGEST) + candidate_workspace_id: str | None = Field(default=None, max_length=256) + candidate_id: str | None = Field(default=None, pattern=_DIGEST) + run_id: str | None = Field(default=None, max_length=256) + decision_id: str | None = Field(default=None, pattern=_DIGEST) + accepted_release_id: str | None = Field(default=None, max_length=256) + stop_reason: EvolutionStopReason | None = None + error_code: EvolutionControllerErrorCode | None = None + + +class EvolutionPolicyRepository(Protocol): + def load( + self, workspace_root: Path, experiment_id: str + ) -> ExperimentPolicySnapshot: ... + + +class EvolutionHypothesisRepository(Protocol): + def load(self, workspace_root: Path, hypothesis_id: str) -> HarnessHypothesis: ... + + +class EvolutionLedger(Protocol): + def events( + self, workspace_root: Path, experiment_id: str + ) -> tuple[EvolutionEvent, ...]: ... + + def append( + self, workspace_root: Path, draft: EvolutionEventDraft + ) -> EvolutionEvent: ... + + +@dataclass(frozen=True, slots=True) +class _EvolutionState: + phase: EvolutionPhase = EvolutionPhase.AWAITING_HYPOTHESIS + iteration: int = 0 + hypothesis_id: str | None = None + candidate_workspace_id: str | None = None + candidate_id: str | None = None + run_id: str | None = None + candidate_receipt_id: str | None = None + decision_id: str | None = None + gate_status: PromotionStatus | None = None + accepted_release_id: str | None = None + stop_reason: EvolutionStopReason | None = None + + +class EvolutionController: + def __init__( + self, + *, + workspace_root: Path, + ledger: EvolutionLedger | None = None, + policy_repository: EvolutionPolicyRepository | None = None, + hypothesis_repository: EvolutionHypothesisRepository | None = None, + ) -> None: + if not workspace_root.is_absolute(): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.POLICY_INVALID, + "workspace_root", + ) + self._workspace_root = workspace_root + self._ledger = ledger or FileEvolutionLedger() + self._policies = policy_repository or FileExperimentPolicyRepository() + self._hypotheses = hypothesis_repository or FileHypothesisRepository() + + def status(self, experiment_id: str) -> EvolutionObservation: + self._policy(experiment_id) + try: + events = self._ledger.events(self._workspace_root, experiment_id) + except EvolutionLedgerFailure as error: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.LEDGER_INVALID, + experiment_id, + ) from error + state = _reduce(events) + return _observation(experiment_id, state, events[-1].sequence if events else 0) + + def advance(self, request: AdvanceEvolutionInput) -> EvolutionObservation: + if request.workspace_root != self._workspace_root: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.REQUEST_CONFLICT, + request.experiment_id, + ) + policy = self._policy(request.experiment_id) + events = self._events(request.experiment_id) + replayed = self._replay_request(request, policy, events) + if replayed is not None: + return replayed + state = _reduce(events) + return self._advance_after_replay(request, policy, state, events) + + def _advance_after_replay( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + events: tuple[EvolutionEvent, ...], + ) -> EvolutionObservation: + if state.phase is EvolutionPhase.STOPPED: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STOPPED, request.experiment_id + ) + special = self._special_advance(request, state) + if special is not None: + return special + if not events: + return self._start(request, policy) + return self._advance_phase(request, policy, state) + + def _events(self, experiment_id: str) -> tuple[EvolutionEvent, ...]: + try: + return self._ledger.events(self._workspace_root, experiment_id) + except EvolutionLedgerFailure as error: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.LEDGER_INVALID, experiment_id + ) from error + + def _replay_request( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + events: tuple[EvolutionEvent, ...], + ) -> EvolutionObservation | None: + prior = _request_event(events, request.request_id) + if prior is None: + return None + if prior.request_digest != request.digest(): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.REQUEST_CONFLICT, request.request_id + ) + resumed = self._resume_after_crash(request, policy, _reduce(events), prior) + if resumed is not None: + return resumed + return _observation(request.experiment_id, _reduce(events), prior.sequence) + + def _special_advance( + self, + request: AdvanceEvolutionInput, + state: _EvolutionState, + ) -> EvolutionObservation | None: + if request.action is EvolutionAdvanceAction.STOP: + return self._stop(request, state) + if request.baseline_deadline_exceeded: + return self._stop_with_reason( + request, state, EvolutionStopReason.BASELINE_DEADLINE + ) + if not request.evidence_available: + return self._stop_with_reason( + request, state, EvolutionStopReason.EVIDENCE_UNAVAILABLE + ) + if request.action is EvolutionAdvanceAction.BLOCK: + return self._block(request, state) + return None + + def _advance_phase( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation: + if state.phase is EvolutionPhase.AWAITING_HYPOTHESIS: + return self._link_hypothesis(request, policy, state) + if state.phase is EvolutionPhase.AWAITING_CANDIDATE: + return self._candidate(request, policy, state) + if state.phase is EvolutionPhase.CANDIDATE_RUNNING: + return self._complete_run(request, state) + return self._advance_gate_or_wait(request, policy, state) + + def _advance_gate_or_wait( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation: + if state.phase is EvolutionPhase.GATE_READY: + return self._decide(request, policy, state) + if state.phase in (EvolutionPhase.AWAITING_PUBLICATION, EvolutionPhase.BLOCKED): + return self._advance_waiting(request, state) + return self._invalid_phase(state) + + def _invalid_phase(self, state: _EvolutionState) -> EvolutionObservation: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, + state.phase.value, + ) + + def _advance_waiting( + self, request: AdvanceEvolutionInput, state: _EvolutionState + ) -> EvolutionObservation: + if state.phase is EvolutionPhase.AWAITING_PUBLICATION: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.PUBLICATION_REQUIRED, + request.experiment_id, + ) + return self._retry(request, state) + + def _resume_after_crash( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + prior: EvolutionEvent, + ) -> EvolutionObservation | None: + if prior.event_type is EvolutionEventType.GATE_DECIDED: + return self._decide(request, policy, state) + if prior.event_type is EvolutionEventType.EXTERNAL_OPERATION_INTENT: + return self._resume_intent(request, policy, state) + return None + + def _resume_intent( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation | None: + if state.phase is EvolutionPhase.AWAITING_CANDIDATE: + return self._candidate(request, policy, state) + if state.phase is EvolutionPhase.CANDIDATE_RUNNING and ( + request.run_id is not None or request.evaluated_run_receipt is not None + ): + return self._complete_run(request, state) + return None + + def _policy(self, experiment_id: str) -> ExperimentPolicySnapshot: + try: + return self._policies.load(self._workspace_root, experiment_id) + except (ExperimentPolicyFailure, ValueError): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.POLICY_INVALID, + experiment_id, + ) from None + + def _start( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + ) -> EvolutionObservation: + self._append( + request, + EvolutionEventType.EVOLUTION_STARTED, + EvolutionStarted(policy_digest=candidate_policy_digest(policy)), + ) + return self.status(request.experiment_id) + + def _link_hypothesis( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation: + hypothesis_id, source_commit = self._validate_hypothesis(request, policy, state) + self._append( + request, + EvolutionEventType.HYPOTHESIS_LINKED, + HypothesisLinked( + hypothesis_id=hypothesis_id, + source_commit=source_commit, + ), + ) + return self.status(request.experiment_id) + + def _validate_hypothesis( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> tuple[str, str]: + if request.hypothesis_id is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.MISSING_INPUT, "hypothesis" + ) + hypothesis_id = request.hypothesis_id + if request.action not in ( + EvolutionAdvanceAction.AUTO, + EvolutionAdvanceAction.LINK_HYPOTHESIS, + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, state.phase.value + ) + hypothesis = self._load_hypothesis(hypothesis_id) + self._validate_hypothesis_identity(request, policy, hypothesis, hypothesis_id) + return hypothesis_id, request.source_commit or hypothesis.source_commit + + def _load_hypothesis(self, hypothesis_id: str) -> HarnessHypothesis: + try: + return self._hypotheses.load(self._workspace_root, hypothesis_id) + except HypothesisFailure: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, hypothesis_id + ) from None + + def _validate_hypothesis_identity( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + hypothesis_id: str, + ) -> None: + source_commit = request.source_commit or hypothesis.source_commit + if ( + hypothesis.experiment_id != request.experiment_id + or source_commit != hypothesis.source_commit + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, hypothesis_id + ) + if source_commit != policy.initialization_commit: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, hypothesis_id + ) + + def _candidate( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation: + if request.candidate_workspace_id is not None: + return self._prepare_candidate(request, policy, state) + return self._submit_candidate(request, state) + + def _prepare_candidate( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation: + workspace_id = request.candidate_workspace_id + if workspace_id is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.MISSING_INPUT, "candidate_workspace" + ) + if state.candidate_workspace_id is not None: + if state.candidate_workspace_id != workspace_id: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.REQUEST_CONFLICT, workspace_id + ) + return self.status(request.experiment_id) + if state.iteration >= policy.max_iterations: + return self._stop_with_reason( + request, state, EvolutionStopReason.MAX_ITERATIONS + ) + key = _operation_key( + request.experiment_id, "candidate-prepare", state.iteration + ) + self._ensure_candidate_intent(request, key, workspace_id) + self._append( + request, + EvolutionEventType.CANDIDATE_PREPARED, + CandidatePrepared( + iteration=state.iteration + 1, candidate_workspace_id=workspace_id + ), + ) + return self.status(request.experiment_id) + + def _ensure_candidate_intent( + self, request: AdvanceEvolutionInput, key: str, target: str + ) -> None: + events = self._ledger.events(self._workspace_root, request.experiment_id) + intent = _find_intent(events, key, ExternalOperation.CANDIDATE) + if intent is not None and intent.target != target: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.REQUEST_CONFLICT, target + ) + if intent is None: + self._append( + request, + EvolutionEventType.EXTERNAL_OPERATION_INTENT, + ExternalOperationIntent( + operation=ExternalOperation.CANDIDATE, + idempotency_key=key, + target=target, + ), + ) + + def _submit_candidate( + self, + request: AdvanceEvolutionInput, + state: _EvolutionState, + ) -> EvolutionObservation: + if request.candidate_id is None or request.candidate_commit is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.MISSING_INPUT, "candidate" + ) + if state.candidate_workspace_id is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, "candidate" + ) + key = _operation_key(request.experiment_id, "candidate", state.iteration) + self._ensure_candidate_intent(request, key, request.candidate_id) + self._append( + request, + EvolutionEventType.CANDIDATE_SUBMITTED, + CandidateSubmitted( + candidate_id=request.candidate_id, + candidate_commit=request.candidate_commit, + ), + ) + return self.status(request.experiment_id) + + def _complete_run( + self, + request: AdvanceEvolutionInput, + state: _EvolutionState, + ) -> EvolutionObservation: + run_id, receipt_id = self._run_details(request, state) + return self._record_run(request, state, run_id, receipt_id) + + def _run_details( + self, + request: AdvanceEvolutionInput, + state: _EvolutionState, + ) -> tuple[str, str]: + self._validate_run_action(request, state) + run_id, receipt_id = _run_values(request) + if state.run_id is not None and state.run_id != run_id: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, run_id + ) + return run_id, receipt_id + + def _validate_run_action( + self, request: AdvanceEvolutionInput, state: _EvolutionState + ) -> None: + if ( + request.candidate_id is not None + and request.candidate_id != state.candidate_id + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, request.candidate_id + ) + if state.candidate_id is None or request.action not in ( + EvolutionAdvanceAction.AUTO, + EvolutionAdvanceAction.COMPLETE_RUN, + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, + state.phase.value, + ) + + def _record_run( + self, + request: AdvanceEvolutionInput, + state: _EvolutionState, + run_id: str, + receipt_id: str, + ) -> EvolutionObservation: + key = _operation_key(request.experiment_id, "harbor", state.iteration) + events = self._ledger.events(self._workspace_root, request.experiment_id) + intent = _find_intent(events, key, ExternalOperation.HARBOR) + if intent is not None and intent.target != run_id: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.REQUEST_CONFLICT, run_id + ) + if intent is None: + self._append( + request, + EvolutionEventType.EXTERNAL_OPERATION_INTENT, + ExternalOperationIntent( + operation=ExternalOperation.HARBOR, + idempotency_key=key, + target=run_id, + ), + ) + self._append( + request, + EvolutionEventType.RUN_COMPLETED, + RunCompleted(run_id=run_id, receipt_id=receipt_id), + ) + return self.status(request.experiment_id) + + def _decide( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> EvolutionObservation: + decision, reasons = self._validate_decision(request, policy, state) + if state.decision_id != decision.decision_id: + self._append( + request, + EvolutionEventType.GATE_DECIDED, + GateDecided( + decision_id=decision.decision_id, + candidate_run_id=decision.candidate_run_id, + status=decision.status, + reasons=reasons, + ), + ) + return self._finish_decision(request, policy, state, decision, reasons) + + def _validate_decision( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> tuple[PromotionDecision, tuple[str, ...]]: + decision = request.promotion_decision + if decision is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.MISSING_INPUT, "promotion_decision" + ) + self._validate_decision_identity(decision, policy, state) + return decision, tuple(reason.value for reason in decision.reasons) + + def _validate_decision_identity( + self, + decision: PromotionDecision, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + ) -> None: + if state.run_id is None or decision.candidate_run_id != state.run_id: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, decision.candidate_run_id + ) + if decision.policy_digest != candidate_policy_digest(policy): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, decision.decision_id + ) + + def _finish_decision( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + decision: PromotionDecision, + reasons: tuple[str, ...], + ) -> EvolutionObservation: + if decision.status is PromotionStatus.INCONCLUSIVE: + return self._block(request, state) + if decision.status is PromotionStatus.ACCEPT: + self._append( + request, + EvolutionEventType.CANDIDATE_ACCEPTED, + CandidateAccepted( + candidate_id=state.candidate_id or "sha256:" + "0" * 64, + decision_id=decision.decision_id, + ), + ) + return self.status(request.experiment_id) + return self._reject_candidate(request, policy, state, decision, reasons) + + def _reject_candidate( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + decision: PromotionDecision, + reasons: tuple[str, ...], + ) -> EvolutionObservation: + self._append( + request, + EvolutionEventType.CANDIDATE_REJECTED, + CandidateRejected( + candidate_id=state.candidate_id or "sha256:" + "0" * 64, + decision_id=decision.decision_id, + reasons=reasons or ("no_improvement",), + ), + ) + rejects = sum( + 1 + for event in self._ledger.events( + self._workspace_root, request.experiment_id + ) + if event.event_type is EvolutionEventType.CANDIDATE_REJECTED + ) + return self._finish_rejection(request, policy, state, reasons, rejects) + + def _finish_rejection( + self, + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + reasons: tuple[str, ...], + rejects: int, + ) -> EvolutionObservation: + stop_reason = _rejection_stop_reason(policy, state, reasons, rejects) + if stop_reason is not None: + return self._stop_with_reason(request, state, stop_reason) + return self.status(request.experiment_id) + + def _retry( + self, request: AdvanceEvolutionInput, state: _EvolutionState + ) -> EvolutionObservation: + if request.action is not EvolutionAdvanceAction.RETRY: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, state.phase.value + ) + run_id = state.run_id or f"run-{state.iteration}" + self._append( + request, + EvolutionEventType.RUN_STARTED, + RunStarted( + run_id=run_id, + idempotency_key=_operation_key( + request.experiment_id, "harbor", state.iteration + ), + ), + ) + return self.status(request.experiment_id) + + def _block( + self, request: AdvanceEvolutionInput, state: _EvolutionState + ) -> EvolutionObservation: + reason = request.blocker_reason or EvolutionStopReason.BLOCKED.value + self._append( + request, + EvolutionEventType.EXTERNAL_OPERATION_BLOCKED, + ExternalOperationBlocked( + operation=ExternalOperation.HARBOR, + idempotency_key=_operation_key( + request.experiment_id, "harbor", state.iteration + ), + reason=reason, + ), + ) + return self.status(request.experiment_id) + + def _stop( + self, request: AdvanceEvolutionInput, state: _EvolutionState + ) -> EvolutionObservation: + return self._stop_with_reason( + request, state, request.stop_reason or EvolutionStopReason.USER_STOP + ) + + def _stop_with_reason( + self, + request: AdvanceEvolutionInput, + state: _EvolutionState, + reason: EvolutionStopReason, + ) -> EvolutionObservation: + del state + self._append( + request, + EvolutionEventType.EVOLUTION_STOPPED, + EvolutionStopped(reason=reason), + ) + return self.status(request.experiment_id) + + def _append( + self, + request: AdvanceEvolutionInput, + event_type: EvolutionEventType, + payload: EvolutionEventPayload, + ) -> EvolutionEvent: + return self._ledger.append( + self._workspace_root, + EvolutionEventDraft( + event_type=event_type, + experiment_id=request.experiment_id, + payload=payload, + occurred_at=request.requested_at, + causation_id=request.request_id, + correlation_id=request.request_id, + request_digest=request.digest(), + ), + ) + + +def _run_values(request: AdvanceEvolutionInput) -> tuple[str, str]: + receipt = request.evaluated_run_receipt + if receipt is not None: + if receipt.side is not RunSide.CANDIDATE: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, receipt.receipt_id + ) + return receipt.run_id, receipt.receipt_id + if request.run_id is None or request.candidate_receipt_id is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.MISSING_INPUT, "run" + ) + return request.run_id, request.candidate_receipt_id + + +def _rejection_stop_reason( + policy: ExperimentPolicySnapshot, + state: _EvolutionState, + reasons: tuple[str, ...], + rejects: int, +) -> EvolutionStopReason | None: + if state.iteration >= policy.max_iterations: + return EvolutionStopReason.MAX_ITERATIONS + if "cost_limit_exceeded" in reasons: + return EvolutionStopReason.COST_LIMIT + if "latency_limit_exceeded" in reasons: + return EvolutionStopReason.LATENCY_LIMIT + if rejects >= policy.no_improvement_limit: + return EvolutionStopReason.NO_IMPROVEMENT + return None + + +def _reduce(events: tuple[EvolutionEvent, ...]) -> _EvolutionState: + state = _EvolutionState() + for event in events: + state = _apply_event(event, state) + return state + + +def _apply_event(event: EvolutionEvent, state: _EvolutionState) -> _EvolutionState: + payload = event.payload + if isinstance(payload, (HypothesisLinked, CandidatePrepared)): + return _apply_candidate_preparation(payload, state) + if isinstance(payload, (CandidateSubmitted, RunStarted)): + return _apply_run_start(payload, state) + if isinstance(payload, (RunCompleted, GateDecided, CandidateAccepted)): + return _apply_progress(payload, state) + if isinstance( + payload, (CandidateRejected, EvolutionStopped, ExternalOperationBlocked) + ): + return _apply_terminal(payload, state) + return state + + +def _apply_candidate_preparation( + payload: HypothesisLinked | CandidatePrepared, state: _EvolutionState +) -> _EvolutionState: + if isinstance(payload, HypothesisLinked): + return _EvolutionState( + EvolutionPhase.AWAITING_CANDIDATE, state.iteration, payload.hypothesis_id + ) + return _EvolutionState( + EvolutionPhase.AWAITING_CANDIDATE, + payload.iteration, + state.hypothesis_id, + payload.candidate_workspace_id, + ) + + +def _apply_run_start( + payload: CandidateSubmitted | RunStarted, state: _EvolutionState +) -> _EvolutionState: + if isinstance(payload, CandidateSubmitted): + return _EvolutionState( + EvolutionPhase.CANDIDATE_RUNNING, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + payload.candidate_id, + ) + return _EvolutionState( + EvolutionPhase.CANDIDATE_RUNNING, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + state.candidate_id, + payload.run_id, + state.candidate_receipt_id, + ) + + +def _apply_progress( + payload: RunCompleted | GateDecided | CandidateAccepted, state: _EvolutionState +) -> _EvolutionState: + if isinstance(payload, RunCompleted): + return _EvolutionState( + EvolutionPhase.GATE_READY, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + state.candidate_id, + payload.run_id, + payload.receipt_id, + ) + if isinstance(payload, GateDecided): + return _EvolutionState( + EvolutionPhase.GATE_READY, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + state.candidate_id, + state.run_id, + state.candidate_receipt_id, + payload.decision_id, + payload.status, + state.accepted_release_id, + ) + return _EvolutionState( + EvolutionPhase.AWAITING_PUBLICATION, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + payload.candidate_id, + state.run_id, + state.candidate_receipt_id, + payload.decision_id, + ) + + +def _apply_terminal( + payload: CandidateRejected | EvolutionStopped | ExternalOperationBlocked, + state: _EvolutionState, +) -> _EvolutionState: + if isinstance(payload, CandidateRejected): + return _EvolutionState(EvolutionPhase.AWAITING_HYPOTHESIS, state.iteration) + if isinstance(payload, EvolutionStopped): + return _EvolutionState( + EvolutionPhase.STOPPED, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + state.candidate_id, + state.run_id, + state.candidate_receipt_id, + state.decision_id, + state.gate_status, + state.accepted_release_id, + payload.reason, + ) + return _EvolutionState( + EvolutionPhase.BLOCKED, + state.iteration, + state.hypothesis_id, + state.candidate_workspace_id, + state.candidate_id, + state.run_id, + state.candidate_receipt_id, + state.decision_id, + state.gate_status, + state.accepted_release_id, + EvolutionStopReason.BLOCKED, + ) + + +def _request_event( + events: tuple[EvolutionEvent, ...], request_id: str +) -> EvolutionEvent | None: + matches = tuple(event for event in events if event.causation_id == request_id) + return matches[0] if matches else None + + +def _find_intent( + events: tuple[EvolutionEvent, ...], + key: str, + operation: ExternalOperation, +) -> ExternalOperationIntent | None: + for event in events: + if _matches_intent(event, key, operation): + payload = event.payload + if isinstance(payload, ExternalOperationIntent): + return payload + return None + + +def _matches_intent( + event: EvolutionEvent, key: str, operation: ExternalOperation +) -> bool: + if event.event_type is not EvolutionEventType.EXTERNAL_OPERATION_INTENT: + return False + if not isinstance(event.payload, ExternalOperationIntent): + return False + return event.payload.idempotency_key == key and event.payload.operation is operation + + +def _observation( + experiment_id: str, + state: _EvolutionState, + sequence: int, +) -> EvolutionObservation: + next_action = { + EvolutionPhase.AWAITING_HYPOTHESIS: "link_hypothesis", + EvolutionPhase.AWAITING_CANDIDATE: "prepare_or_submit_candidate", + EvolutionPhase.CANDIDATE_RUNNING: "complete_candidate_run", + EvolutionPhase.GATE_READY: "record_promotion_decision", + EvolutionPhase.AWAITING_PUBLICATION: "publish_accepted_candidate", + EvolutionPhase.BLOCKED: "retry_after_external_state_change", + EvolutionPhase.STOPPED: "stop", + }[state.phase] + status = ( + EvolutionStatus.ERROR + if state.phase is EvolutionPhase.STOPPED + else EvolutionStatus.SUCCESS + ) + return EvolutionObservation( + experiment_id=experiment_id, + status=status, + phase=state.phase, + summary=f"Evolution is {state.phase.value}.", + next_actions=(next_action,), + sequence=sequence, + iteration=state.iteration, + hypothesis_id=state.hypothesis_id, + candidate_workspace_id=state.candidate_workspace_id, + candidate_id=state.candidate_id, + run_id=state.run_id, + decision_id=state.decision_id, + accepted_release_id=state.accepted_release_id, + stop_reason=state.stop_reason, + ) + + +def _operation_key(experiment_id: str, operation: str, iteration: int) -> str: + value = f"{experiment_id}\0{operation}\0{iteration}" + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() diff --git a/src/ofw/evolution/ledger.py b/src/ofw/evolution/ledger.py new file mode 100644 index 0000000..b625f73 --- /dev/null +++ b/src/ofw/evolution/ledger.py @@ -0,0 +1,655 @@ +"""Append-only, typed evolution events in Git's common control directory.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import os +import re +import stat +from collections.abc import Iterator +from contextlib import contextmanager, suppress +from dataclasses import dataclass +from datetime import datetime, timedelta +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Literal, TypeAlias +from uuid import uuid4 + +from pydantic import Field, field_validator, model_validator + +from ofw.evolution.gate import PromotionStatus +from ofw.preparation.contracts import StrictModel +from ofw.preparation.policy import ExperimentPolicyFailure, experiment_control_directory +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_child_directory, + open_directory_chain, + read_bounded, + write_new_file, +) + +_DIGEST = r"sha256:[0-9a-f]{64}" +_IDENTIFIER = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" +_EXPERIMENT = r"[a-z0-9]+(?:-[a-z0-9]+)*" +_COMMIT = r"[0-9a-f]{40}" +_LEDGER_LIMIT_BYTES = 4 * 1024 * 1024 +_EVENT_LIMIT_BYTES = 256 * 1024 + +Identifier = Annotated[str, Field(min_length=1, max_length=256, pattern=_IDENTIFIER)] +Digest = Annotated[str, Field(pattern=_DIGEST)] + + +class EvolutionEventType(StrEnum): + EVOLUTION_STARTED = "EvolutionStarted" + HYPOTHESIS_LINKED = "HypothesisLinked" + CANDIDATE_PREPARED = "CandidatePrepared" + CANDIDATE_SUBMITTED = "CandidateSubmitted" + RUN_STARTED = "RunStarted" + RUN_COMPLETED = "RunCompleted" + GATE_DECIDED = "GateDecided" + CANDIDATE_ACCEPTED = "CandidateAccepted" + CANDIDATE_REJECTED = "CandidateRejected" + RELEASE_PUBLISHED = "ReleasePublished" + RELEASE_ROLLED_BACK = "ReleaseRolledBack" + EVOLUTION_STOPPED = "EvolutionStopped" + EXTERNAL_OPERATION_INTENT = "ExternalOperationIntent" + EXTERNAL_OPERATION_BLOCKED = "ExternalOperationBlocked" + + +class ExternalOperation(StrEnum): + CANDIDATE = "candidate" + HARBOR = "harbor" + PUBLICATION = "publication" + + +class EvolutionStopReason(StrEnum): + QUALITY_TARGET = "quality_target" + MAX_ITERATIONS = "max_iterations" + NO_IMPROVEMENT = "no_improvement" + COST_LIMIT = "cost_limit" + LATENCY_LIMIT = "latency_limit" + BASELINE_DEADLINE = "baseline_deadline" + EVIDENCE_UNAVAILABLE = "evidence_unavailable" + USER_STOP = "user_stop" + BLOCKED = "blocked" + + +class EvolutionStarted(StrictModel): + policy_digest: Digest + + +class HypothesisLinked(StrictModel): + hypothesis_id: Digest + source_commit: str = Field(pattern=_COMMIT) + + +class CandidatePrepared(StrictModel): + iteration: int = Field(strict=True, ge=1, le=100) + candidate_workspace_id: Identifier + + +class CandidateSubmitted(StrictModel): + candidate_id: Digest + candidate_commit: str = Field(pattern=_COMMIT) + + +class RunStarted(StrictModel): + run_id: Identifier + idempotency_key: Digest + + +class RunCompleted(StrictModel): + run_id: Identifier + receipt_id: Digest + + +class GateDecided(StrictModel): + decision_id: Digest + candidate_run_id: Identifier + status: PromotionStatus + reasons: tuple[Identifier, ...] = Field(max_length=20) + + +class CandidateAccepted(StrictModel): + candidate_id: Digest + decision_id: Digest + + +class CandidateRejected(StrictModel): + candidate_id: Digest + decision_id: Digest + reasons: tuple[Identifier, ...] = Field(max_length=20) + + +class ReleasePublished(StrictModel): + release_id: Identifier + + +class ReleaseRolledBack(StrictModel): + release_id: Identifier + target_release_id: Identifier + + +class EvolutionStopped(StrictModel): + reason: EvolutionStopReason + + +class ExternalOperationIntent(StrictModel): + operation: ExternalOperation + idempotency_key: Digest + target: Identifier + + +class ExternalOperationBlocked(StrictModel): + operation: ExternalOperation + idempotency_key: Digest + reason: Identifier + + +EvolutionEventPayload: TypeAlias = ( + EvolutionStarted + | HypothesisLinked + | CandidatePrepared + | CandidateSubmitted + | RunStarted + | RunCompleted + | GateDecided + | CandidateAccepted + | CandidateRejected + | ReleasePublished + | ReleaseRolledBack + | EvolutionStopped + | ExternalOperationIntent + | ExternalOperationBlocked +) + + +class EvolutionLedgerErrorCode(StrEnum): + INVALID_EVENT = "invalid_event" + INVALID_WORKSPACE = "invalid_workspace" + BUSY = "busy" + EVENT_CONFLICT = "event_conflict" + CORRUPT_LEDGER = "corrupt_ledger" + SEQUENCE_GAP = "sequence_gap" + CURSOR_INVALID = "cursor_invalid" + LEDGER_TOO_LARGE = "ledger_too_large" + WRITE_FAILED = "write_failed" + + +class EvolutionLedgerFailure(Exception): + __slots__ = ("code", "subject", "last_valid_sequence") + + def __init__( + self, + code: EvolutionLedgerErrorCode, + subject: str, + last_valid_sequence: int = 0, + ) -> None: + self.code = code + self.subject = subject + self.last_valid_sequence = last_valid_sequence + super().__init__(f"{code.value}: {subject}") + + +class EvolutionEvent(StrictModel): + schema_version: Literal[1] = 1 + experiment_id: str = Field(pattern=_EXPERIMENT, max_length=80) + sequence: int = Field(strict=True, ge=1) + event_id: Digest + event_type: EvolutionEventType + occurred_at: datetime + causation_id: Identifier | None = None + correlation_id: Identifier | None = None + request_digest: Digest | None = None + payload: EvolutionEventPayload + + @field_validator("occurred_at") + @classmethod + def validate_utc(cls, value: datetime) -> datetime: + if value.utcoffset() != timedelta(0): + raise ValueError("occurred_at must be UTC") + return value + + @model_validator(mode="after") + def validate_payload_type(self) -> EvolutionEvent: + payload_types: tuple[tuple[EvolutionEventType, type[object]], ...] = ( + (EvolutionEventType.EVOLUTION_STARTED, EvolutionStarted), + (EvolutionEventType.HYPOTHESIS_LINKED, HypothesisLinked), + (EvolutionEventType.CANDIDATE_PREPARED, CandidatePrepared), + (EvolutionEventType.CANDIDATE_SUBMITTED, CandidateSubmitted), + (EvolutionEventType.RUN_STARTED, RunStarted), + (EvolutionEventType.RUN_COMPLETED, RunCompleted), + (EvolutionEventType.GATE_DECIDED, GateDecided), + (EvolutionEventType.CANDIDATE_ACCEPTED, CandidateAccepted), + (EvolutionEventType.CANDIDATE_REJECTED, CandidateRejected), + (EvolutionEventType.RELEASE_PUBLISHED, ReleasePublished), + (EvolutionEventType.RELEASE_ROLLED_BACK, ReleaseRolledBack), + (EvolutionEventType.EVOLUTION_STOPPED, EvolutionStopped), + (EvolutionEventType.EXTERNAL_OPERATION_INTENT, ExternalOperationIntent), + (EvolutionEventType.EXTERNAL_OPERATION_BLOCKED, ExternalOperationBlocked), + ) + for event_type, payload_type in payload_types: + if self.event_type is event_type: + if not isinstance(self.payload, payload_type): + raise ValueError("event payload does not match event_type") + return self + raise ValueError("unknown event type") + + def fingerprint(self) -> str: + content = self.model_dump_json(exclude={"sequence", "event_id"}) + return _digest(content) + + +@dataclass(frozen=True, slots=True) +class EvolutionEventDraft: + event_type: EvolutionEventType + experiment_id: str + payload: EvolutionEventPayload + occurred_at: datetime + causation_id: str | None = None + correlation_id: str | None = None + request_digest: str | None = None + event_id: str | None = None + + def __post_init__(self) -> None: + if re.fullmatch(_EXPERIMENT, self.experiment_id) is None: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_EVENT, "experiment_id" + ) + if self.occurred_at.utcoffset() != timedelta(0): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_EVENT, "occurred_at" + ) + + def build(self, sequence: int) -> EvolutionEvent: + content = EvolutionEvent( + experiment_id=self.experiment_id, + sequence=sequence, + event_id="sha256:" + "0" * 64, + event_type=self.event_type, + occurred_at=self.occurred_at, + causation_id=self.causation_id, + correlation_id=self.correlation_id, + request_digest=self.request_digest, + payload=self.payload, + ) + identity = _draft_identity(self, content) + event_id = self.event_id or _digest(identity) + return EvolutionEvent( + experiment_id=content.experiment_id, + sequence=content.sequence, + event_id=event_id, + event_type=content.event_type, + occurred_at=content.occurred_at, + causation_id=content.causation_id, + correlation_id=content.correlation_id, + request_digest=content.request_digest, + payload=content.payload, + ) + + +@dataclass(frozen=True, slots=True) +class EvolutionEventPage: + events: tuple[EvolutionEvent, ...] + next_cursor: str | None + + +class FileEvolutionLedger: + """One append-only event log per experiment, rooted at the Git common dir.""" + + def append( + self, workspace_root: Path, draft: EvolutionEventDraft + ) -> EvolutionEvent: + control = _control(workspace_root, draft.experiment_id) + try: + with _writer(control) as directory: + return _append_to_directory(directory, draft) + except EvolutionLedgerFailure: + raise + except SafeFileFailure as error: + raise _safe_failure(error, draft.experiment_id) from None + except OSError: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.WRITE_FAILED, + draft.experiment_id, + ) from None + + def page( + self, + workspace_root: Path, + experiment_id: str, + *, + cursor: str | None = None, + limit: int = 20, + ) -> EvolutionEventPage: + if not 1 <= limit <= 500: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_EVENT, "limit" + ) + control = _control(workspace_root, experiment_id) + try: + with open_directory_chain( + control.parents[2], + ("ofw", "preparations", control.name), + create=False, + ) as directory: + events = _read_events(directory, experiment_id) + except FileNotFoundError: + events = () + except SafeFileFailure as error: + raise _safe_failure(error, experiment_id) from None + except OSError: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_WORKSPACE, + experiment_id, + ) from None + return _select_page(events, experiment_id, cursor, limit) + + def events( + self, workspace_root: Path, experiment_id: str + ) -> tuple[EvolutionEvent, ...]: + events: tuple[EvolutionEvent, ...] = () + cursor: str | None = None + while True: + page = self.page( + workspace_root, + experiment_id, + cursor=cursor, + limit=500, + ) + events += page.events + if page.next_cursor is None: + return events + cursor = page.next_cursor + + +def _append_to_directory(directory: int, draft: EvolutionEventDraft) -> EvolutionEvent: + events = _read_events(directory, draft.experiment_id) + candidate = draft.build(len(events) + 1) + for existing in events: + if existing.event_id != candidate.event_id: + continue + if existing.fingerprint() == candidate.fingerprint(): + return existing + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.EVENT_CONFLICT, + candidate.event_id, + existing.sequence, + ) + _append_event(directory, candidate) + return candidate + + +def _select_page( + events: tuple[EvolutionEvent, ...], + experiment_id: str, + cursor: str | None, + limit: int, +) -> EvolutionEventPage: + after = _decode_cursor(cursor, experiment_id) if cursor is not None else 0 + selected = tuple(event for event in events if event.sequence > after) + page = selected[:limit] + next_cursor = _next_cursor(experiment_id, selected, page) + return EvolutionEventPage(page, next_cursor) + + +def _next_cursor( + experiment_id: str, + selected: tuple[EvolutionEvent, ...], + page: tuple[EvolutionEvent, ...], +) -> str | None: + if len(selected) <= len(page) or not page: + return None + return _encode_cursor(experiment_id, page[-1].sequence) + + +@contextmanager +def _writer(control: Path) -> Iterator[int]: + with open_directory_chain( + control.parents[2], + ("ofw", "preparations", control.name), + create=True, + ) as directory: + try: + os.mkdir(".evolution.lock", 0o700, dir_fd=directory) + except FileExistsError: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.BUSY, control.name + ) from None + token = uuid4().hex.encode("ascii") + try: + with open_child_directory( + directory, ".evolution.lock", create=False + ) as lock: + write_new_file(lock, "owner", token) + if ( + read_bounded(lock, "owner", maximum_bytes=64, subject=control.name) + != token + ): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.BUSY, control.name + ) + try: + yield directory + if ( + read_bounded( + lock, "owner", maximum_bytes=64, subject=control.name + ) + != token + ): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.BUSY, control.name + ) + finally: + with suppress(FileNotFoundError): + os.unlink("owner", dir_fd=lock) + finally: + with suppress(FileNotFoundError): + os.rmdir(".evolution.lock", dir_fd=directory) + os.fsync(directory) + + +def _control(root: Path, experiment_id: str) -> Path: + if not root.is_absolute() or re.fullmatch(_EXPERIMENT, experiment_id) is None: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_WORKSPACE, experiment_id + ) + try: + return experiment_control_directory(root, experiment_id) + except ExperimentPolicyFailure: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_WORKSPACE, + experiment_id, + ) from None + + +def _read_events(directory: int, experiment_id: str) -> tuple[EvolutionEvent, ...]: + try: + content = read_bounded( + directory, + "evolution.jsonl", + maximum_bytes=_LEDGER_LIMIT_BYTES, + subject=experiment_id, + ) + except FileNotFoundError: + return () + except SafeFileFailure as error: + if error.code is SafeFileErrorCode.TOO_LARGE: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.LEDGER_TOO_LARGE, + experiment_id, + ) from None + raise + if not content: + return () + return _parse_events(content, experiment_id) + + +def _parse_events(content: bytes, experiment_id: str) -> tuple[EvolutionEvent, ...]: + events: list[EvolutionEvent] = [] + for line in content.splitlines(keepends=True): + event = _parse_event_line(line, experiment_id, events) + _validate_event_order(event, experiment_id, events) + events.append(event) + return tuple(events) + + +def _parse_event_line( + line: bytes, experiment_id: str, events: list[EvolutionEvent] +) -> EvolutionEvent: + last = events[-1].sequence if events else 0 + if not line.endswith(b"\n"): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CORRUPT_LEDGER, experiment_id, last + ) + try: + return EvolutionEvent.model_validate_json(line) + except (ValueError, UnicodeError): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CORRUPT_LEDGER, experiment_id, last + ) from None + + +def _validate_event_order( + event: EvolutionEvent, experiment_id: str, events: list[EvolutionEvent] +) -> None: + last = events[-1].sequence if events else 0 + if event.experiment_id != experiment_id: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CORRUPT_LEDGER, experiment_id, last + ) + if event.sequence != last + 1: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.SEQUENCE_GAP, experiment_id, last + ) + if _has_duplicate_event(events, event.event_id): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CORRUPT_LEDGER, experiment_id, last + ) + _validate_event_identity(event, experiment_id, last) + + +def _validate_event_identity( + event: EvolutionEvent, experiment_id: str, last: int +) -> None: + identity = _event_identity(event) + if event.event_id != _digest(identity): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CORRUPT_LEDGER, experiment_id, last + ) + + +def _has_duplicate_event(events: list[EvolutionEvent], event_id: str) -> bool: + return any(item.event_id == event_id for item in events) + + +def _draft_identity(draft: EvolutionEventDraft, event: EvolutionEvent) -> str: + fingerprint = ( + event.fingerprint() + if draft.causation_id is None and draft.correlation_id is None + else "" + ) + return "\0".join( + ( + draft.experiment_id, + draft.event_type.value, + draft.causation_id or "", + draft.correlation_id or "", + fingerprint, + ) + ) + + +def _event_identity(event: EvolutionEvent) -> str: + fingerprint = ( + event.fingerprint() + if event.causation_id is None and event.correlation_id is None + else "" + ) + return "\0".join( + ( + event.experiment_id, + event.event_type.value, + event.causation_id or "", + event.correlation_id or "", + fingerprint, + ) + ) + + +def _append_event(directory: int, event: EvolutionEvent) -> None: + content = (event.model_dump_json() + "\n").encode("utf-8") + if len(content) > _EVENT_LIMIT_BYTES: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.LEDGER_TOO_LARGE, + event.event_id, + event.sequence - 1, + ) + flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = os.open("evolution.jsonl", flags, 0o600, dir_fd=directory) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.INVALID_WORKSPACE, + event.experiment_id, + event.sequence - 1, + ) + view = memoryview(content) + while view: + view = view[os.write(descriptor, view) :] + os.fsync(descriptor) + finally: + os.close(descriptor) + os.fsync(directory) + + +def _encode_cursor(experiment_id: str, sequence: int) -> str: + body = f"1\0{experiment_id}\0{sequence}".encode() + digest = hashlib.sha256(body).hexdigest()[:32].encode("ascii") + return base64.urlsafe_b64encode(body + b"\0" + digest).decode("ascii").rstrip("=") + + +def _decode_cursor(value: str, experiment_id: str) -> int: + try: + raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + version, actual_experiment, sequence_text, digest = raw.split(b"\0") + return _cursor_sequence(version, actual_experiment, sequence_text, digest, experiment_id) + except (ValueError, UnicodeError, binascii.Error): + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CURSOR_INVALID, + experiment_id, + ) from None + + +def _cursor_sequence( + version: bytes, + actual_experiment: bytes, + sequence_text: bytes, + digest: bytes, + experiment_id: str, +) -> int: + body = b"\0".join((version, actual_experiment, sequence_text)) + if version != b"1" or actual_experiment.decode() != experiment_id: + raise ValueError + if digest != hashlib.sha256(body).hexdigest()[:32].encode("ascii"): + raise ValueError + sequence = int(sequence_text) + if sequence < 0: + raise ValueError + return sequence + + +def _digest(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _safe_failure(error: SafeFileFailure, subject: str) -> EvolutionLedgerFailure: + code = ( + EvolutionLedgerErrorCode.LEDGER_TOO_LARGE + if error.code is SafeFileErrorCode.TOO_LARGE + else EvolutionLedgerErrorCode.INVALID_WORKSPACE + ) + return EvolutionLedgerFailure(code, subject) diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index 9bdb0fc..f6c58fa 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -9,6 +9,7 @@ from datetime import datetime from enum import StrEnum from importlib.resources import files +from pathlib import Path from typing import Annotated, TypeVar from mcp.server.fastmcp import FastMCP @@ -46,10 +47,13 @@ VerifierVerdict, ) from ofw.evolution import ( + AdvanceEvolutionInput, CandidateExecutionInput, CandidateExecutionObservation, CandidateExecutionService, CandidateGitGateway, + EvolutionController, + EvolutionObservation, FileHypothesisRepository, HypothesisObservation, HypothesisService, @@ -88,10 +92,15 @@ CursorIdentifier = Annotated[str, Field(min_length=1, max_length=4096)] TracePageLimit = Annotated[int, Field(strict=True, ge=1, le=50)] TaskIdentifier = Annotated[str, Field(min_length=1, max_length=256)] +EvolutionExperimentIdentifier = Annotated[ + str, Field(min_length=1, max_length=80, pattern=r"[a-z0-9]+(?:-[a-z0-9]+)*") +] VerifierIdentifier = Annotated[str, Field(min_length=1, max_length=256)] OutcomeScore = Annotated[float, Field(strict=True, ge=0.0, le=1.0)] EvidenceIdentifier = Annotated[str, Field(min_length=1, max_length=1024)] -OutcomeEvidence = Annotated[tuple[EvidenceIdentifier, ...], Field(min_length=1, max_length=10)] +OutcomeEvidence = Annotated[ + tuple[EvidenceIdentifier, ...], Field(min_length=1, max_length=10) +] server = FastMCP[None]( # type: ignore[misc] # MCP auth generics are untyped upstream. name="openflywheel", @@ -195,6 +204,10 @@ def _candidate_service() -> Iterator[CandidateExecutionService]: client.close() +def _evolution_controller(workspace_root: Path) -> EvolutionController: + return EvolutionController(workspace_root=workspace_root) + + def _program_template(name: str) -> str: content = files("ofw.preparation.templates").joinpath(name).read_bytes() if len(content) > _PROGRAM_TEMPLATE_LIMIT_BYTES: @@ -351,6 +364,20 @@ def execute_candidate( return service.execute(request) +@server.tool(annotations=read_only, structured_output=True) +def evolution_status( + workspace_root: Path, experiment_id: EvolutionExperimentIdentifier +) -> EvolutionObservation: + """Read the replayed evolution state without appending or mutating it.""" + return _evolution_controller(workspace_root).status(experiment_id) + + +@server.tool(annotations=record_write, structured_output=True) +def advance_evolution(request: AdvanceEvolutionInput) -> EvolutionObservation: + """Advance one deterministic evolution action; identical requests are idempotent.""" + return _evolution_controller(request.workspace_root).advance(request) + + def main() -> None: """Run the OpenFlywheel MCP server over stdio.""" server.run(transport="stdio") diff --git a/src/ofw/preparation/__init__.py b/src/ofw/preparation/__init__.py index 96ce3d6..8135abd 100644 --- a/src/ofw/preparation/__init__.py +++ b/src/ofw/preparation/__init__.py @@ -23,6 +23,7 @@ ExperimentPolicyFailure, ExperimentPolicySnapshot, FileExperimentPolicyRepository, + experiment_control_directory, ) from ofw.preparation.service import WorkspacePreparationService @@ -39,6 +40,7 @@ "ExperimentPolicyFailure", "ExperimentPolicySnapshot", "FileExperimentPolicyRepository", + "experiment_control_directory", "PreparationErrorCode", "PreparationFailure", "PreparationPhase", diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py index 7f6929a..f4409bf 100644 --- a/src/ofw/preparation/policy.py +++ b/src/ofw/preparation/policy.py @@ -236,6 +236,11 @@ def _snapshot_from_content(content: _ExperimentPolicyContent) -> ExperimentPolic def _control_directory(workspace_root: Path, experiment_id: str) -> Path: + return experiment_control_directory(workspace_root, experiment_id) + + +def experiment_control_directory(workspace_root: Path, experiment_id: str) -> Path: + """Return the experiment directory in Git's common control area.""" result = subprocess.run( ("git", "-C", str(workspace_root), "rev-parse", "--git-common-dir"), check=False, diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index c440f02..eb2f5d2 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -4,8 +4,8 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Record one evidence-backed hypothesis, execute its isolated candidate under the canonical -experiment policy, then stop before admission. +Call `evolution_status`, then `advance_evolution` for exactly one next action under the +canonical experiment policy. Stop before admission. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an @@ -31,38 +31,26 @@ Target only exact paths allowed by the canonical experiment policy. Edit them on candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks, verifier, model, reasoning budget, observability identity, or this program. -Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control -flow changes unless the evidence requires the combination. - -## Optimization loop - -### 2. Analyze failures - -Start from verifier-backed failed outcomes. Use bounded trace queries to locate relevant -evidence, then inspect only the spans needed to explain the observed behavior. Do not load -or copy complete traces when filters answer the question, and do not inspect held-out -trajectory content. - -### 3. Form one hypothesis - -State the failure pattern, supporting trace and verifier evidence, proposed harness change, -expected improvement, and possible regressions. Stop if the evidence cannot distinguish -between materially different changes. - -Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported -pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call -`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution. - -### 4. Execute one candidate - -Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling -candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated -worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the -returned candidate worktree, then call `execute_candidate` again with the identical request. - -Poll identical requests while the candidate is running. Retain its candidate ID, Git commit, -trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally, -change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree. +Keep one focused hypothesis per iteration. The controller owns phase, budgets, stop reasons, +and ledger truth; never append events or perform generic transitions yourself. + +## Controller loop + +1. Use bounded trace queries and `$failure-miner`, `record_failure`, `$failure-pattern-miner`, + `mine_failure_patterns`, `$failure-curator`, and `record_failure_curation` to retain only + typed evidence; do not copy Langfuse trace payloads. +2. Use `$hypothesis-former` and `record_hypothesis`, then pass the stable hypothesis receipt to + `advance_evolution`. +3. When the controller returns a candidate worktree, edit only its declared targets and use the + returned candidate worktree with `execute_candidate`; retain the candidate and evaluated run receipts. +4. Pass the existing `PromotionDecision` to `advance_evolution`. Accepted candidates remain + `AwaitingPublication` until PR5; do not publish, merge, push, or install. + +The controller returns the required next action. Use bounded trace queries and the failure +skills to retain only typed evidence; do not copy Langfuse trace payloads. Form one hypothesis +with `$hypothesis-former` and `record_hypothesis`, then pass its receipt to `advance_evolution`. +Call `execute_candidate` when requested, edit only its declared targets, and repeat the identical +request. Pass the existing `PromotionDecision` to `advance_evolution`. ## Package boundary diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/support_policy.py b/tests/support_policy.py new file mode 100644 index 0000000..5b74551 --- /dev/null +++ b/tests/support_policy.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from pathlib import Path + +from ofw.contracts import ComponentKind +from ofw.evolution.hypothesis import ( + HarnessChangeTarget, + HarnessHypothesis, + HypothesisId, +) +from ofw.preparation.policy import ExperimentPolicySnapshot + + +def policy( + *, max_iterations: int = 2, no_improvement_limit: int = 1 +) -> ExperimentPolicySnapshot: + draft = ExperimentPolicySnapshot.model_construct( + experiment_id="experiment-one", + branch_name="ofw/experiment-one", + base_commit="a" * 40, + initialization_commit="a" * 40, + editable_paths=(Path("PROGRAM.md"),), + goal="Improve quality", + quality_target=1.0, + max_iterations=max_iterations, + no_improvement_limit=no_improvement_limit, + max_baseline_seconds=60, + benchmark_config_digest="sha256:" + "b" * 64, + task_ids=("task-1",), + model="model", + verifier="verifier", + environment="test", + controls_digest="sha256:" + "0" * 64, + ) + # Pydantic's model_construct is the only way to bootstrap this self-hashed fixture. + return draft.model_copy( + update={"controls_digest": draft.recomputed_controls_digest()} # type: ignore[misc] + ) + + +class PolicyRepository: + def __init__(self, value: ExperimentPolicySnapshot) -> None: + self.value = value + + def load( + self, workspace_root: Path, experiment_id: str + ) -> ExperimentPolicySnapshot: + del workspace_root, experiment_id + return self.value + + +class HypothesisRepository: + def load(self, workspace_root: Path, hypothesis_id: str) -> HarnessHypothesis: + del workspace_root + return HarnessHypothesis( + id=HypothesisId(hypothesis_id), + experiment_id="experiment-one", + source_commit="a" * 40, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", + patterns=(), + statement="statement", + rationale="rationale", + target=HarnessChangeTarget(ComponentKind.SKILL, (Path("PROGRAM.md"),)), + expected_effect="effect", + regression_risks=(), + ) diff --git a/tests/test_evolution_controller.py b/tests/test_evolution_controller.py new file mode 100644 index 0000000..91b9e57 --- /dev/null +++ b/tests/test_evolution_controller.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.evolution.candidate import candidate_policy_digest +from ofw.evolution.controller import ( + AdvanceEvolutionInput, + EvolutionAdvanceAction, + EvolutionController, + EvolutionControllerErrorCode, + EvolutionControllerFailure, + EvolutionPhase, + EvolutionStatus, + EvolutionStopReason, +) +from ofw.evolution.gate import PromotionDecision, PromotionReason, PromotionStatus +from ofw.evolution.ledger import ( + EvolutionEvent, + EvolutionEventDraft, + EvolutionLedgerErrorCode, + EvolutionLedgerFailure, + FileEvolutionLedger, +) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, +) +from tests.support_policy import PolicyRepository, policy + + +def _repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + root.mkdir(parents=True) + subprocess.run(("git", "-C", str(root), "init", "-q"), check=True) + return root + + +def _controller( + root: Path, + *, + max_iterations: int = 2, + no_improvement_limit: int = 1, +) -> EvolutionController: + from tests.support_policy import HypothesisRepository, PolicyRepository, policy + + return EvolutionController( + workspace_root=root, + ledger=FileEvolutionLedger(), + policy_repository=PolicyRepository( + policy( + max_iterations=max_iterations, no_improvement_limit=no_improvement_limit + ) + ), + hypothesis_repository=HypothesisRepository(), + ) + + +def _request( + root: Path, + request_id: str, + *, + action: EvolutionAdvanceAction = EvolutionAdvanceAction.AUTO, + hypothesis_id: str | None = None, + candidate_workspace_id: str | None = None, + candidate_id: str | None = None, + candidate_commit: str | None = None, + run_id: str | None = None, + candidate_receipt_id: str | None = None, + promotion_decision: PromotionDecision | None = None, + release_id: str | None = None, + stop_reason: EvolutionStopReason | None = None, + blocker_reason: str | None = None, + baseline_deadline_exceeded: bool = False, + evidence_available: bool = True, +) -> AdvanceEvolutionInput: + return AdvanceEvolutionInput( + workspace_root=root, + experiment_id="experiment-one", + request_id=request_id, + action=action, + hypothesis_id=hypothesis_id, + candidate_workspace_id=candidate_workspace_id, + candidate_id=candidate_id, + candidate_commit=candidate_commit, + run_id=run_id, + candidate_receipt_id=candidate_receipt_id, + promotion_decision=promotion_decision, + release_id=release_id, + stop_reason=stop_reason, + blocker_reason=blocker_reason, + baseline_deadline_exceeded=baseline_deadline_exceeded, + evidence_available=evidence_available, + ) + + +def test_controller_advances_one_deterministic_step_and_retries_idempotently( + tmp_path: Path, +) -> None: + root = _repo(tmp_path) + controller = _controller(root) + initial = controller.status("experiment-one") + assert initial.phase is EvolutionPhase.AWAITING_HYPOTHESIS + started = controller.advance(_request(root, "r1")) + assert started.phase is EvolutionPhase.AWAITING_HYPOTHESIS + assert started.status is EvolutionStatus.SUCCESS + assert controller.advance(_request(root, "r1")) == started + + linked = controller.advance( + _request(root, "r2", hypothesis_id="sha256:" + "a" * 64) + ) + assert linked.phase is EvolutionPhase.AWAITING_CANDIDATE + + +def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + controller.advance(_request(root, "r1")) + controller.advance(_request(root, "r2", hypothesis_id="sha256:" + "a" * 64)) + controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) + controller.advance( + _request( + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + ) + ) + running = controller.advance( + _request(root, "r5", run_id="run-1", candidate_receipt_id="sha256:" + "c" * 64) + ) + assert running.phase is EvolutionPhase.GATE_READY + decision = PromotionDecision( + decision_id="sha256:" + "d" * 64, + policy_digest=candidate_policy_digest(policy()), + accepted_run_id="baseline", + candidate_run_id="run-1", + status=PromotionStatus.ACCEPT, + reasons=(PromotionReason.IMPROVEMENT,), + task_ids=("task-1",), + accepted_passes=(), + candidate_passes=("task-1",), + accepted_quality=0.0, + candidate_quality=1.0, + accepted_cost_usd=None, + candidate_cost_usd=None, + accepted_latency_seconds=None, + candidate_latency_seconds=None, + canonical_json="{}", + ) + accepted = controller.advance(_request(root, "r6", promotion_decision=decision)) + assert accepted.phase is EvolutionPhase.AWAITING_PUBLICATION + from ofw.evolution.controller import ( + EvolutionControllerErrorCode, + EvolutionControllerFailure, + ) + + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance(_request(root, "r7", release_id="release-1")) + assert raised.value.code is EvolutionControllerErrorCode.PUBLICATION_REQUIRED + assert ( + controller.status("experiment-one").phase is EvolutionPhase.AWAITING_PUBLICATION + ) + + +def test_explicit_stop_is_typed_and_terminal(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + stopped = controller.advance( + _request( + root, + "stop", + action=EvolutionAdvanceAction.STOP, + stop_reason=EvolutionStopReason.USER_STOP, + ) + ) + assert stopped.phase is EvolutionPhase.STOPPED + assert stopped.stop_reason is EvolutionStopReason.USER_STOP + assert controller.status("experiment-one") == stopped + + +def test_input_and_workspace_boundaries_are_strict(tmp_path: Path) -> None: + with pytest.raises(ValidationError): + AdvanceEvolutionInput( + workspace_root=Path("relative"), + experiment_id="experiment-one", + request_id="request-1", + ) + root = _repo(tmp_path) + controller = _controller(root) + request = _request(root / "other", "request-1") + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance(request) + assert raised.value.code is EvolutionControllerErrorCode.REQUEST_CONFLICT + with pytest.raises(ValidationError): + AdvanceEvolutionInput( + workspace_root=root, + experiment_id="experiment-one", + request_id="stop-without-reason", + action=EvolutionAdvanceAction.STOP, + ) + + +def test_baseline_and_evidence_stops_are_durable(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + baseline = controller.advance( + _request(root, "baseline", baseline_deadline_exceeded=True) + ) + assert baseline.phase is EvolutionPhase.STOPPED + assert baseline.stop_reason is EvolutionStopReason.BASELINE_DEADLINE + + root = _repo(tmp_path / "second") + controller = _controller(root) + evidence = controller.advance(_request(root, "evidence", evidence_available=False)) + assert evidence.stop_reason is EvolutionStopReason.EVIDENCE_UNAVAILABLE + + +def test_missing_and_stale_hypothesis_receipts_fail_closed(tmp_path: Path) -> None: + root = _repo(tmp_path) + from tests.support_policy import PolicyRepository + + controller = EvolutionController( + workspace_root=root, + ledger=FileEvolutionLedger(), + policy_repository=PolicyRepository(policy()), + ) + controller.advance(_request(root, "start")) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request(root, "missing", hypothesis_id="sha256:" + "a" * 64) + ) + assert raised.value.code is EvolutionControllerErrorCode.STALE_RECEIPT + + +def test_missing_candidate_input_fails_without_ledger_mutation(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + controller.advance(_request(root, "start")) + controller.advance(_request(root, "hypothesis", hypothesis_id="sha256:" + "a" * 64)) + before = controller.status("experiment-one").sequence + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance(_request(root, "candidate")) + assert raised.value.code is EvolutionControllerErrorCode.MISSING_INPUT + assert controller.status("experiment-one").sequence == before + + +def test_block_retry_and_conflicting_external_targets(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + controller.advance(_request(root, "r1")) + controller.advance(_request(root, "r2", hypothesis_id="sha256:" + "a" * 64)) + controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) + controller.advance( + _request( + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + ) + ) + blocked = controller.advance( + _request( + root, + "r5", + action=EvolutionAdvanceAction.BLOCK, + blocker_reason="provider_timeout", + ) + ) + assert blocked.phase is EvolutionPhase.BLOCKED + retried = controller.advance( + _request(root, "r6", action=EvolutionAdvanceAction.RETRY) + ) + assert retried.phase is EvolutionPhase.CANDIDATE_RUNNING + + root = _repo(tmp_path / "conflict") + controller = _controller(root) + controller.advance(_request(root, "r1")) + controller.advance(_request(root, "r2", hypothesis_id="sha256:" + "a" * 64)) + controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) + controller.advance( + _request( + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + ) + ) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request( + root, "r5", candidate_id="sha256:" + "c" * 64, candidate_commit="a" * 40 + ) + ) + assert raised.value.code is EvolutionControllerErrorCode.STALE_RECEIPT + + +def test_max_iterations_and_no_improvement_stops(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root, max_iterations=1, no_improvement_limit=1) + controller.advance(_request(root, "r1")) + controller.advance(_request(root, "r2", hypothesis_id="sha256:" + "a" * 64)) + controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) + controller.advance( + _request( + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + ) + ) + controller.advance( + _request(root, "r5", run_id="run-1", candidate_receipt_id="sha256:" + "c" * 64) + ) + decision = PromotionDecision( + decision_id="sha256:" + "d" * 64, + policy_digest=candidate_policy_digest( + policy(max_iterations=1, no_improvement_limit=1) + ), + accepted_run_id="baseline", + candidate_run_id="run-1", + status=PromotionStatus.REJECT, + reasons=(PromotionReason.NO_IMPROVEMENT,), + task_ids=("task-1",), + accepted_passes=("task-1",), + candidate_passes=("task-1",), + accepted_quality=1.0, + candidate_quality=1.0, + accepted_cost_usd=None, + candidate_cost_usd=None, + accepted_latency_seconds=None, + candidate_latency_seconds=None, + canonical_json="{}", + ) + stopped = controller.advance(_request(root, "r6", promotion_decision=decision)) + assert stopped.stop_reason is EvolutionStopReason.MAX_ITERATIONS + + +def test_policy_and_ledger_failures_are_sanitized(tmp_path: Path) -> None: + root = _repo(tmp_path) + + class FailingPolicy: + def load( + self, workspace_root: Path, experiment_id: str + ) -> ExperimentPolicySnapshot: + del workspace_root, experiment_id + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, "secret-path" + ) + + with pytest.raises(EvolutionControllerFailure) as raised: + EvolutionController( + workspace_root=root, + policy_repository=FailingPolicy(), + ).status("experiment-one") + assert raised.value.code is EvolutionControllerErrorCode.POLICY_INVALID + assert "secret-path" not in str(raised.value) + + class FailingLedger: + def events( + self, workspace_root: Path, experiment_id: str + ) -> tuple[EvolutionEvent, ...]: + del workspace_root, experiment_id + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.CORRUPT_LEDGER, "secret-content", 4 + ) + + def append( + self, workspace_root: Path, draft: EvolutionEventDraft + ) -> EvolutionEvent: + del workspace_root, draft + raise AssertionError("append must not be reached") + + with pytest.raises(EvolutionControllerFailure) as raised: + EvolutionController( + workspace_root=root, + ledger=FailingLedger(), + policy_repository=PolicyRepository(policy()), + ).status("experiment-one") + assert raised.value.code is EvolutionControllerErrorCode.LEDGER_INVALID + assert "secret-content" not in str(raised.value) + + with pytest.raises(EvolutionControllerFailure) as raised: + EvolutionController( + workspace_root=root, + ledger=FailingLedger(), + policy_repository=PolicyRepository(policy()), + ).advance(_request(root, "advance")) + assert raised.value.code is EvolutionControllerErrorCode.LEDGER_INVALID + + +def test_conflicting_controller_retry_is_rejected(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + controller.advance(_request(root, "same")) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request( + root, + "same", + action=EvolutionAdvanceAction.STOP, + stop_reason=EvolutionStopReason.USER_STOP, + ) + ) + assert raised.value.code is EvolutionControllerErrorCode.REQUEST_CONFLICT + + +def test_illegal_action_and_stopped_state_are_typed(tmp_path: Path) -> None: + root = _repo(tmp_path) + controller = _controller(root) + controller.advance(_request(root, "r1")) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request( + root, + "bad", + action=EvolutionAdvanceAction.PREPARE_CANDIDATE, + hypothesis_id="sha256:" + "a" * 64, + ) + ) + assert raised.value.code is EvolutionControllerErrorCode.INVALID_TRANSITION + stopped = controller.advance( + _request( + root, + "stop", + action=EvolutionAdvanceAction.STOP, + stop_reason=EvolutionStopReason.USER_STOP, + ) + ) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance(_request(root, "after")) + assert raised.value.code is EvolutionControllerErrorCode.STOPPED + assert stopped.phase is EvolutionPhase.STOPPED diff --git a/tests/test_evolution_ledger.py b/tests/test_evolution_ledger.py new file mode 100644 index 0000000..b0d1931 --- /dev/null +++ b/tests/test_evolution_ledger.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.evolution.ledger import ( + EvolutionEvent, + EvolutionEventDraft, + EvolutionEventType, + EvolutionLedgerErrorCode, + EvolutionLedgerFailure, + EvolutionStarted, + FileEvolutionLedger, + HypothesisLinked, +) + + +def _repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + root.mkdir() + (root / "file").write_text("x", encoding="utf-8") + import subprocess + + subprocess.run(("git", "-C", str(root), "init", "-q"), check=True) + return root + + +def _draft(experiment_id: str = "experiment-one") -> EvolutionEventDraft: + return EvolutionEventDraft( + event_type=EvolutionEventType.EVOLUTION_STARTED, + experiment_id=experiment_id, + payload=EvolutionStarted(policy_digest="sha256:" + "a" * 64), + occurred_at=datetime(2026, 9, 3, tzinfo=UTC), + correlation_id="request-1", + ) + + +def test_append_is_fsynced_typed_and_keyset_paginated(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + + first = ledger.append(root, _draft()) + second = ledger.append( + root, + EvolutionEventDraft( + event_type=EvolutionEventType.EVOLUTION_STARTED, + experiment_id="experiment-one", + payload=EvolutionStarted(policy_digest="sha256:" + "b" * 64), + occurred_at=datetime(2026, 9, 3, 0, 0, 1, tzinfo=UTC), + correlation_id="request-2", + ), + ) + assert (first.sequence, second.sequence) == (1, 2) + page = ledger.page(root, "experiment-one", limit=1) + assert page.events == (first,) + assert page.next_cursor is not None + assert ledger.page( + root, "experiment-one", cursor=page.next_cursor, limit=1 + ).events == (second,) + + +def test_empty_log_and_invalid_draft_inputs_are_handled(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + assert ledger.page(root, "experiment-one").events == () + with pytest.raises(EvolutionLedgerFailure): + EvolutionEventDraft( + event_type=EvolutionEventType.EVOLUTION_STARTED, + experiment_id="bad/id", + payload=EvolutionStarted(policy_digest="sha256:" + "a" * 64), + occurred_at=datetime(2026, 9, 3, tzinfo=UTC), + ) + with pytest.raises(EvolutionLedgerFailure): + EvolutionEventDraft( + event_type=EvolutionEventType.EVOLUTION_STARTED, + experiment_id="experiment-one", + payload=EvolutionStarted(policy_digest="sha256:" + "a" * 64), + occurred_at=datetime(2026, 9, 3), + ) + with pytest.raises(ValidationError): + EvolutionEvent( + experiment_id="experiment-one", + sequence=1, + event_id="sha256:" + "a" * 64, + event_type=EvolutionEventType.EVOLUTION_STARTED, + occurred_at=datetime(2026, 9, 3), + payload=EvolutionStarted(policy_digest="sha256:" + "a" * 64), + ) + + +def test_identical_retry_is_idempotent_and_conflict_fails(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + first = ledger.append(root, _draft()) + assert ledger.append(root, _draft()) == first + + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.append( + root, + EvolutionEventDraft( + event_type=EvolutionEventType.EVOLUTION_STARTED, + experiment_id="experiment-one", + payload=EvolutionStarted(policy_digest="sha256:" + "c" * 64), + occurred_at=datetime(2026, 9, 3, tzinfo=UTC), + correlation_id="request-1", + ), + ) + assert raised.value.code is EvolutionLedgerErrorCode.EVENT_CONFLICT + + +def test_corrupt_tail_fails_closed_with_last_valid_sequence(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + ledger.append(root, _draft()) + path = root / ".git" / "ofw" / "preparations" / "experiment-one" / "evolution.jsonl" + with path.open("ab") as stream: + stream.write(b'{"schema_version":1') + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.page(root, "experiment-one") + assert raised.value.code is EvolutionLedgerErrorCode.CORRUPT_LEDGER + assert raised.value.last_valid_sequence == 1 + + +def test_gapped_sequence_fails_closed(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + ledger.append(root, _draft()) + path = root / ".git" / "ofw" / "preparations" / "experiment-one" / "evolution.jsonl" + content = path.read_text(encoding="utf-8").replace('"sequence":1', '"sequence":3') + path.write_text(content, encoding="utf-8") + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.page(root, "experiment-one") + assert raised.value.code is EvolutionLedgerErrorCode.SEQUENCE_GAP + assert raised.value.last_valid_sequence == 0 + + +def test_cursor_and_count_bounds_are_typed(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + ledger.append(root, _draft()) + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.page(root, "experiment-one", limit=0) + assert raised.value.code is EvolutionLedgerErrorCode.INVALID_EVENT + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.page(root, "experiment-one", cursor="not-a-cursor") + assert raised.value.code is EvolutionLedgerErrorCode.CURSOR_INVALID + + +def test_invalid_workspace_and_payload_fail_closed(tmp_path: Path) -> None: + with pytest.raises(EvolutionLedgerFailure) as raised: + FileEvolutionLedger().page(tmp_path / "missing", "experiment-one") + assert raised.value.code is EvolutionLedgerErrorCode.INVALID_WORKSPACE + with pytest.raises(ValidationError): + EvolutionEvent( + experiment_id="experiment-one", + sequence=1, + event_id="sha256:" + "a" * 64, + event_type=EvolutionEventType.EVOLUTION_STARTED, + occurred_at=datetime(2026, 9, 3, tzinfo=UTC), + payload=HypothesisLinked( + hypothesis_id="sha256:" + "a" * 64, + source_commit="a" * 40, + ), + ) + + +def test_writer_owner_lock_is_exclusive(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + ledger.append(root, _draft()) + lock = root / ".git" / "ofw" / "preparations" / "experiment-one" / ".evolution.lock" + lock.mkdir() + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.append(root, _draft("experiment-one")) + assert raised.value.code is EvolutionLedgerErrorCode.BUSY + lock.rmdir() + + +def test_append_rejects_non_regular_log_and_oversized_log(tmp_path: Path) -> None: + root = _repo(tmp_path) + ledger = FileEvolutionLedger() + control = root / ".git" / "ofw" / "preparations" / "experiment-one" + control.mkdir(parents=True) + (control / "evolution.jsonl").mkdir() + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.append(root, _draft()) + assert raised.value.code is EvolutionLedgerErrorCode.INVALID_WORKSPACE + + (control / "evolution.jsonl").rmdir() + (control / "evolution.jsonl").write_bytes(b"x" * (4 * 1024 * 1024 + 1)) + with pytest.raises(EvolutionLedgerFailure) as raised: + ledger.page(root, "experiment-one") + assert raised.value.code is EvolutionLedgerErrorCode.LEDGER_TOO_LARGE diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index 5d931df..8f6026d 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -419,6 +419,8 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: "record_failure_curation", "record_hypothesis", "execute_candidate", + "evolution_status", + "advance_evolution", ] assert tuple(map(_annotation_flags, tools)) == ( (False, False, True), @@ -432,6 +434,8 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: (False, False, True), (False, False, True), (False, False, True), + (True, False, True), + (False, False, True), ) From fc9a8f17d07513158fcd803c2e6cf8d8ed9bbd7b Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:06:27 +0530 Subject: [PATCH 05/11] fix: track accepted evolution content identity --- src/ofw/evolution/controller.py | 235 +++++++++++++++++++---------- src/ofw/evolution/ledger.py | 17 ++- tests/support_policy.py | 5 +- tests/test_evolution_controller.py | 113 +++++++++++++- 4 files changed, 285 insertions(+), 85 deletions(-) diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py index e10a8c7..877c59e 100644 --- a/src/ofw/evolution/controller.py +++ b/src/ofw/evolution/controller.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime from enum import StrEnum from pathlib import Path @@ -35,6 +35,8 @@ FileEvolutionLedger, GateDecided, HypothesisLinked, + ReleasePublished, + ReleaseRolledBack, RunCompleted, RunStarted, ) @@ -122,6 +124,11 @@ class AdvanceEvolutionInput(StrictModel): action: EvolutionAdvanceAction = EvolutionAdvanceAction.AUTO hypothesis_id: str | None = Field(default=None, pattern=_DIGEST) source_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + accepted_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + accepted_content_id: str | None = Field(default=None, pattern=_DIGEST) + accepted_release_id: str | None = Field( + default=None, max_length=256, pattern=_IDENTIFIER + ) candidate_workspace_id: str | None = Field( default=None, max_length=256, pattern=_IDENTIFIER ) @@ -167,6 +174,8 @@ class EvolutionObservation(StrictModel): run_id: str | None = Field(default=None, max_length=256) decision_id: str | None = Field(default=None, pattern=_DIGEST) accepted_release_id: str | None = Field(default=None, max_length=256) + accepted_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + accepted_content_id: str | None = Field(default=None, pattern=_DIGEST) stop_reason: EvolutionStopReason | None = None error_code: EvolutionControllerErrorCode | None = None @@ -204,6 +213,9 @@ class _EvolutionState: gate_status: PromotionStatus | None = None accepted_release_id: str | None = None stop_reason: EvolutionStopReason | None = None + accepted_commit: str | None = None + accepted_content_id: str | None = None + candidate_commit: str | None = None class EvolutionController: @@ -226,7 +238,7 @@ def __init__( self._hypotheses = hypothesis_repository or FileHypothesisRepository() def status(self, experiment_id: str) -> EvolutionObservation: - self._policy(experiment_id) + policy = self._policy(experiment_id) try: events = self._ledger.events(self._workspace_root, experiment_id) except EvolutionLedgerFailure as error: @@ -234,7 +246,7 @@ def status(self, experiment_id: str) -> EvolutionObservation: EvolutionControllerErrorCode.LEDGER_INVALID, experiment_id, ) from error - state = _reduce(events) + state = _accepted_identity(_reduce(events), policy) return _observation(experiment_id, state, events[-1].sequence if events else 0) def advance(self, request: AdvanceEvolutionInput) -> EvolutionObservation: @@ -397,10 +409,19 @@ def _start( request: AdvanceEvolutionInput, policy: ExperimentPolicySnapshot, ) -> EvolutionObservation: + accepted_commit = request.accepted_commit or policy.initialization_commit + accepted_content_id = request.accepted_content_id or _content_identity( + accepted_commit + ) self._append( request, EvolutionEventType.EVOLUTION_STARTED, - EvolutionStarted(policy_digest=candidate_policy_digest(policy)), + EvolutionStarted( + policy_digest=candidate_policy_digest(policy), + accepted_commit=accepted_commit, + accepted_content_id=accepted_content_id, + accepted_release_id=request.accepted_release_id, + ), ) return self.status(request.experiment_id) @@ -440,7 +461,10 @@ def _validate_hypothesis( EvolutionControllerErrorCode.INVALID_TRANSITION, state.phase.value ) hypothesis = self._load_hypothesis(hypothesis_id) - self._validate_hypothesis_identity(request, policy, hypothesis, hypothesis_id) + expected_commit = state.accepted_commit or policy.initialization_commit + self._validate_hypothesis_identity( + request, expected_commit, hypothesis, hypothesis_id + ) return hypothesis_id, request.source_commit or hypothesis.source_commit def _load_hypothesis(self, hypothesis_id: str) -> HarnessHypothesis: @@ -454,7 +478,7 @@ def _load_hypothesis(self, hypothesis_id: str) -> HarnessHypothesis: def _validate_hypothesis_identity( self, request: AdvanceEvolutionInput, - policy: ExperimentPolicySnapshot, + expected_commit: str, hypothesis: HarnessHypothesis, hypothesis_id: str, ) -> None: @@ -466,7 +490,7 @@ def _validate_hypothesis_identity( raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, hypothesis_id ) - if source_commit != policy.initialization_commit: + if source_commit != expected_commit: raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, hypothesis_id ) @@ -502,6 +526,8 @@ def _prepare_candidate( return self._stop_with_reason( request, state, EvolutionStopReason.MAX_ITERATIONS ) + source_commit = state.accepted_commit or policy.initialization_commit + source_content_id = state.accepted_content_id or _content_identity(source_commit) key = _operation_key( request.experiment_id, "candidate-prepare", state.iteration ) @@ -510,7 +536,11 @@ def _prepare_candidate( request, EvolutionEventType.CANDIDATE_PREPARED, CandidatePrepared( - iteration=state.iteration + 1, candidate_workspace_id=workspace_id + iteration=state.iteration + 1, + candidate_workspace_id=workspace_id, + source_commit=source_commit, + source_content_id=source_content_id, + source_release_id=state.accepted_release_id, ), ) return self.status(request.experiment_id) @@ -697,6 +727,8 @@ def _finish_decision( CandidateAccepted( candidate_id=state.candidate_id or "sha256:" + "0" * 64, decision_id=decision.decision_id, + candidate_commit=state.candidate_commit, + accepted_content_id=state.candidate_id, ), ) return self.status(request.experiment_id) @@ -860,6 +892,15 @@ def _reduce(events: tuple[EvolutionEvent, ...]) -> _EvolutionState: def _apply_event(event: EvolutionEvent, state: _EvolutionState) -> _EvolutionState: payload = event.payload + if isinstance(payload, EvolutionStarted): + return replace( + state, + accepted_commit=payload.accepted_commit, + accepted_content_id=payload.accepted_content_id, + accepted_release_id=payload.accepted_release_id, + ) + if isinstance(payload, (ReleasePublished, ReleaseRolledBack)): + return _apply_release(payload, state) if isinstance(payload, (HypothesisLinked, CandidatePrepared)): return _apply_candidate_preparation(payload, state) if isinstance(payload, (CandidateSubmitted, RunStarted)): @@ -873,18 +914,49 @@ def _apply_event(event: EvolutionEvent, state: _EvolutionState) -> _EvolutionSta return state +def _apply_release( + payload: ReleasePublished | ReleaseRolledBack, state: _EvolutionState +) -> _EvolutionState: + if isinstance(payload, ReleasePublished): + target_reached = payload.target_reached + return replace( + state, + phase=( + EvolutionPhase.STOPPED + if target_reached + else EvolutionPhase.AWAITING_HYPOTHESIS + ), + accepted_release_id=payload.release_id, + accepted_commit=payload.content_commit or state.accepted_commit, + accepted_content_id=payload.content_id or state.accepted_content_id, + stop_reason=( + EvolutionStopReason.QUALITY_TARGET if target_reached else None + ), + ) + return replace( + state, + phase=EvolutionPhase.AWAITING_HYPOTHESIS, + accepted_release_id=payload.release_id, + accepted_commit=payload.content_commit or state.accepted_commit, + accepted_content_id=payload.content_id or state.accepted_content_id, + stop_reason=None, + ) + + def _apply_candidate_preparation( payload: HypothesisLinked | CandidatePrepared, state: _EvolutionState ) -> _EvolutionState: if isinstance(payload, HypothesisLinked): - return _EvolutionState( - EvolutionPhase.AWAITING_CANDIDATE, state.iteration, payload.hypothesis_id + return replace( + state, + phase=EvolutionPhase.AWAITING_CANDIDATE, + hypothesis_id=payload.hypothesis_id, ) - return _EvolutionState( - EvolutionPhase.AWAITING_CANDIDATE, - payload.iteration, - state.hypothesis_id, - payload.candidate_workspace_id, + return replace( + state, + phase=EvolutionPhase.AWAITING_CANDIDATE, + iteration=payload.iteration, + candidate_workspace_id=payload.candidate_workspace_id, ) @@ -892,21 +964,16 @@ def _apply_run_start( payload: CandidateSubmitted | RunStarted, state: _EvolutionState ) -> _EvolutionState: if isinstance(payload, CandidateSubmitted): - return _EvolutionState( - EvolutionPhase.CANDIDATE_RUNNING, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - payload.candidate_id, + return replace( + state, + phase=EvolutionPhase.CANDIDATE_RUNNING, + candidate_id=payload.candidate_id, + candidate_commit=payload.candidate_commit, ) - return _EvolutionState( - EvolutionPhase.CANDIDATE_RUNNING, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - state.candidate_id, - payload.run_id, - state.candidate_receipt_id, + return replace( + state, + phase=EvolutionPhase.CANDIDATE_RUNNING, + run_id=payload.run_id, ) @@ -914,37 +981,27 @@ def _apply_progress( payload: RunCompleted | GateDecided | CandidateAccepted, state: _EvolutionState ) -> _EvolutionState: if isinstance(payload, RunCompleted): - return _EvolutionState( - EvolutionPhase.GATE_READY, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - state.candidate_id, - payload.run_id, - payload.receipt_id, + return replace( + state, + phase=EvolutionPhase.GATE_READY, + run_id=payload.run_id, + candidate_receipt_id=payload.receipt_id, ) if isinstance(payload, GateDecided): - return _EvolutionState( - EvolutionPhase.GATE_READY, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - state.candidate_id, - state.run_id, - state.candidate_receipt_id, - payload.decision_id, - payload.status, - state.accepted_release_id, + return replace( + state, + phase=EvolutionPhase.GATE_READY, + decision_id=payload.decision_id, + gate_status=payload.status, ) - return _EvolutionState( - EvolutionPhase.AWAITING_PUBLICATION, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - payload.candidate_id, - state.run_id, - state.candidate_receipt_id, - payload.decision_id, + return replace( + state, + phase=EvolutionPhase.AWAITING_PUBLICATION, + candidate_id=payload.candidate_id, + decision_id=payload.decision_id, + candidate_commit=payload.candidate_commit or state.candidate_commit, + accepted_commit=payload.candidate_commit or state.candidate_commit, + accepted_content_id=payload.accepted_content_id or payload.candidate_id, ) @@ -953,33 +1010,28 @@ def _apply_terminal( state: _EvolutionState, ) -> _EvolutionState: if isinstance(payload, CandidateRejected): - return _EvolutionState(EvolutionPhase.AWAITING_HYPOTHESIS, state.iteration) + return replace( + state, + phase=EvolutionPhase.AWAITING_HYPOTHESIS, + hypothesis_id=None, + candidate_workspace_id=None, + candidate_id=None, + candidate_commit=None, + run_id=None, + candidate_receipt_id=None, + decision_id=None, + gate_status=None, + ) if isinstance(payload, EvolutionStopped): - return _EvolutionState( - EvolutionPhase.STOPPED, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - state.candidate_id, - state.run_id, - state.candidate_receipt_id, - state.decision_id, - state.gate_status, - state.accepted_release_id, - payload.reason, + return replace( + state, + phase=EvolutionPhase.STOPPED, + stop_reason=payload.reason, ) - return _EvolutionState( - EvolutionPhase.BLOCKED, - state.iteration, - state.hypothesis_id, - state.candidate_workspace_id, - state.candidate_id, - state.run_id, - state.candidate_receipt_id, - state.decision_id, - state.gate_status, - state.accepted_release_id, - EvolutionStopReason.BLOCKED, + return replace( + state, + phase=EvolutionPhase.BLOCKED, + stop_reason=EvolutionStopReason.BLOCKED, ) @@ -1046,6 +1098,8 @@ def _observation( run_id=state.run_id, decision_id=state.decision_id, accepted_release_id=state.accepted_release_id, + accepted_commit=state.accepted_commit, + accepted_content_id=state.accepted_content_id, stop_reason=state.stop_reason, ) @@ -1053,3 +1107,20 @@ def _observation( def _operation_key(experiment_id: str, operation: str, iteration: int) -> str: value = f"{experiment_id}\0{operation}\0{iteration}" return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _content_identity(commit: str) -> str: + return "sha256:" + hashlib.sha256(f"git-commit\0{commit}".encode()).hexdigest() + + +def _accepted_identity( + state: _EvolutionState, policy: ExperimentPolicySnapshot +) -> _EvolutionState: + if state.accepted_commit is not None and state.accepted_content_id is not None: + return state + commit = state.accepted_commit or policy.initialization_commit + return replace( + state, + accepted_commit=commit, + accepted_content_id=state.accepted_content_id or _content_identity(commit), + ) diff --git a/src/ofw/evolution/ledger.py b/src/ofw/evolution/ledger.py index b625f73..1d0f9cc 100644 --- a/src/ofw/evolution/ledger.py +++ b/src/ofw/evolution/ledger.py @@ -79,6 +79,9 @@ class EvolutionStopReason(StrEnum): class EvolutionStarted(StrictModel): policy_digest: Digest + accepted_commit: str | None = Field(default=None, pattern=_COMMIT) + accepted_content_id: Digest | None = None + accepted_release_id: Identifier | None = None class HypothesisLinked(StrictModel): @@ -89,6 +92,9 @@ class HypothesisLinked(StrictModel): class CandidatePrepared(StrictModel): iteration: int = Field(strict=True, ge=1, le=100) candidate_workspace_id: Identifier + source_commit: str | None = Field(default=None, pattern=_COMMIT) + source_content_id: Digest | None = None + source_release_id: Identifier | None = None class CandidateSubmitted(StrictModel): @@ -116,6 +122,8 @@ class GateDecided(StrictModel): class CandidateAccepted(StrictModel): candidate_id: Digest decision_id: Digest + candidate_commit: str | None = Field(default=None, pattern=_COMMIT) + accepted_content_id: Digest | None = None class CandidateRejected(StrictModel): @@ -126,11 +134,16 @@ class CandidateRejected(StrictModel): class ReleasePublished(StrictModel): release_id: Identifier + content_commit: str | None = Field(default=None, pattern=_COMMIT) + content_id: Digest | None = None + target_reached: bool = False class ReleaseRolledBack(StrictModel): release_id: Identifier target_release_id: Identifier + content_commit: str | None = Field(default=None, pattern=_COMMIT) + content_id: Digest | None = None class EvolutionStopped(StrictModel): @@ -616,7 +629,9 @@ def _decode_cursor(value: str, experiment_id: str) -> int: try: raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) version, actual_experiment, sequence_text, digest = raw.split(b"\0") - return _cursor_sequence(version, actual_experiment, sequence_text, digest, experiment_id) + return _cursor_sequence( + version, actual_experiment, sequence_text, digest, experiment_id + ) except (ValueError, UnicodeError, binascii.Error): raise EvolutionLedgerFailure( EvolutionLedgerErrorCode.CURSOR_INVALID, diff --git a/tests/support_policy.py b/tests/support_policy.py index 5b74551..a08866a 100644 --- a/tests/support_policy.py +++ b/tests/support_policy.py @@ -50,12 +50,15 @@ def load( class HypothesisRepository: + def __init__(self, source_commit: str = "a" * 40) -> None: + self.source_commit = source_commit + def load(self, workspace_root: Path, hypothesis_id: str) -> HarnessHypothesis: del workspace_root return HarnessHypothesis( id=HypothesisId(hypothesis_id), experiment_id="experiment-one", - source_commit="a" * 40, + source_commit=self.source_commit, curation_id="00000000-0000-0000-0000-000000000001", curation_group_id="00000000-0000-0000-0000-000000000002", patterns=(), diff --git a/tests/test_evolution_controller.py b/tests/test_evolution_controller.py index 91b9e57..45d22f5 100644 --- a/tests/test_evolution_controller.py +++ b/tests/test_evolution_controller.py @@ -1,6 +1,7 @@ from __future__ import annotations import subprocess +from datetime import UTC, datetime from pathlib import Path import pytest @@ -19,11 +20,15 @@ ) from ofw.evolution.gate import PromotionDecision, PromotionReason, PromotionStatus from ofw.evolution.ledger import ( + CandidatePrepared, EvolutionEvent, EvolutionEventDraft, + EvolutionEventType, EvolutionLedgerErrorCode, EvolutionLedgerFailure, FileEvolutionLedger, + ReleasePublished, + ReleaseRolledBack, ) from ofw.preparation.policy import ( ExperimentPolicyErrorCode, @@ -66,6 +71,10 @@ def _request( *, action: EvolutionAdvanceAction = EvolutionAdvanceAction.AUTO, hypothesis_id: str | None = None, + source_commit: str | None = None, + accepted_commit: str | None = None, + accepted_content_id: str | None = None, + accepted_release_id: str | None = None, candidate_workspace_id: str | None = None, candidate_id: str | None = None, candidate_commit: str | None = None, @@ -84,6 +93,10 @@ def _request( request_id=request_id, action=action, hypothesis_id=hypothesis_id, + source_commit=source_commit, + accepted_commit=accepted_commit, + accepted_content_id=accepted_content_id, + accepted_release_id=accepted_release_id, candidate_workspace_id=candidate_workspace_id, candidate_id=candidate_id, candidate_commit=candidate_commit, @@ -151,6 +164,10 @@ def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) - ) accepted = controller.advance(_request(root, "r6", promotion_decision=decision)) assert accepted.phase is EvolutionPhase.AWAITING_PUBLICATION + assert (accepted.accepted_commit, accepted.accepted_content_id) == ( + "a" * 40, + "sha256:" + "b" * 64, + ) from ofw.evolution.controller import ( EvolutionControllerErrorCode, EvolutionControllerFailure, @@ -221,9 +238,10 @@ def test_missing_and_stale_hypothesis_receipts_fail_closed(tmp_path: Path) -> No root = _repo(tmp_path) from tests.support_policy import PolicyRepository + ledger = FileEvolutionLedger() controller = EvolutionController( workspace_root=root, - ledger=FileEvolutionLedger(), + ledger=ledger, policy_repository=PolicyRepository(policy()), ) controller.advance(_request(root, "start")) @@ -423,3 +441,96 @@ def test_illegal_action_and_stopped_state_are_typed(tmp_path: Path) -> None: controller.advance(_request(root, "after")) assert raised.value.code is EvolutionControllerErrorCode.STOPPED assert stopped.phase is EvolutionPhase.STOPPED + + +def test_current_accepted_identity_is_replayed_and_exposed(tmp_path: Path) -> None: + root = _repo(tmp_path) + accepted_commit = "b" * 40 + accepted_content_id = "sha256:" + "c" * 64 + from tests.support_policy import HypothesisRepository, PolicyRepository + + ledger = FileEvolutionLedger() + controller = EvolutionController( + workspace_root=root, + ledger=ledger, + policy_repository=PolicyRepository(policy()), + hypothesis_repository=HypothesisRepository(accepted_commit), + ) + started = controller.advance( + _request( + root, + "start", + accepted_commit=accepted_commit, + accepted_content_id=accepted_content_id, + accepted_release_id="release-a", + ) + ) + assert (started.accepted_commit, started.accepted_content_id) == ( + accepted_commit, + accepted_content_id, + ) + controller.advance( + _request( + root, + "hypothesis", + hypothesis_id="sha256:" + "a" * 64, + source_commit=accepted_commit, + ) + ) + prepared = controller.advance( + _request(root, "candidate", candidate_workspace_id="workspace-1") + ) + event = ledger.events(root, "experiment-one")[-1] + assert isinstance(event.payload, CandidatePrepared) + assert (event.payload.source_commit, event.payload.source_content_id) == ( + accepted_commit, + accepted_content_id, + ) + assert (prepared.accepted_commit, prepared.accepted_content_id) == ( + accepted_commit, + accepted_content_id, + ) + + ledger.append( + root, + EvolutionEventDraft( + event_type=EvolutionEventType.RELEASE_PUBLISHED, + experiment_id="experiment-one", + payload=ReleasePublished( + release_id="release-b", + content_commit="d" * 40, + content_id="sha256:" + "e" * 64, + target_reached=True, + ), + occurred_at=datetime(2026, 9, 3, 0, 0, 1, tzinfo=UTC), + causation_id="publish-b", + correlation_id="publish-b", + ), + ) + published = controller.status("experiment-one") + assert (published.phase, published.stop_reason, published.accepted_release_id) == ( + EvolutionPhase.STOPPED, + EvolutionStopReason.QUALITY_TARGET, + "release-b", + ) + ledger.append( + root, + EvolutionEventDraft( + event_type=EvolutionEventType.RELEASE_ROLLED_BACK, + experiment_id="experiment-one", + payload=ReleaseRolledBack( + release_id="release-c", + target_release_id="release-a", + content_commit=accepted_commit, + content_id=accepted_content_id, + ), + occurred_at=datetime(2026, 9, 3, 0, 0, 2, tzinfo=UTC), + causation_id="rollback-c", + correlation_id="rollback-c", + ), + ) + rolled_back = controller.status("experiment-one") + assert (rolled_back.phase, rolled_back.accepted_commit) == ( + EvolutionPhase.AWAITING_HYPOTHESIS, + accepted_commit, + ) From dd7ecf2e2b48874a88cab11c04a0f25f92fd39de Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:18:13 +0530 Subject: [PATCH 06/11] fix: harden evolution review boundaries --- .../openflywheel/program_templates/base.md | 14 +-- src/ofw/evolution/controller.py | 97 ++++++++++++++----- src/ofw/evolution/gate.py | 55 ++++++++--- src/ofw/evolution/ledger.py | 80 ++++++++------- src/ofw/mcp.py | 22 ++++- src/ofw/preparation/policy.py | 1 + src/ofw/preparation/templates/base.md | 14 +-- tests/test_evolution_controller.py | 17 +++- tests/test_evolution_ledger.py | 4 +- tests/test_program_templates.py | 37 +++++-- 10 files changed, 237 insertions(+), 104 deletions(-) diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index eb2f5d2..b4ab777 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -41,18 +41,12 @@ and ledger truth; never append events or perform generic transitions yourself. typed evidence; do not copy Langfuse trace payloads. 2. Use `$hypothesis-former` and `record_hypothesis`, then pass the stable hypothesis receipt to `advance_evolution`. -3. When the controller returns a candidate worktree, edit only its declared targets and use the - returned candidate worktree with `execute_candidate`; retain the candidate and evaluated run receipts. +3. Call `execute_candidate` to create the candidate worktree, edit only its declared targets, + then call it again with the identical request; retain candidate and evaluated run receipts. 4. Pass the existing `PromotionDecision` to `advance_evolution`. Accepted candidates remain `AwaitingPublication` until PR5; do not publish, merge, push, or install. -The controller returns the required next action. Use bounded trace queries and the failure -skills to retain only typed evidence; do not copy Langfuse trace payloads. Form one hypothesis -with `$hypothesis-former` and `record_hypothesis`, then pass its receipt to `advance_evolution`. -Call `execute_candidate` when requested, edit only its declared targets, and repeat the identical -request. Pass the existing `PromotionDecision` to `advance_evolution`. - ## Package boundary -Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission: -do not gate, accept, merge, publish, push, or install the candidate. +Report the hypothesis, candidate, commit, blocker, gate, and outcome receipts. Stop before +publication: do not publish, merge, push, or install the candidate. diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py index 877c59e..eb7d051 100644 --- a/src/ofw/evolution/controller.py +++ b/src/ofw/evolution/controller.py @@ -312,6 +312,14 @@ def _special_advance( request: AdvanceEvolutionInput, state: _EvolutionState, ) -> EvolutionObservation | None: + if ( + state.phase is EvolutionPhase.AWAITING_PUBLICATION + and request.action is EvolutionAdvanceAction.BLOCK + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.PUBLICATION_REQUIRED, + request.experiment_id, + ) if request.action is EvolutionAdvanceAction.STOP: return self._stop(request, state) if request.baseline_deadline_exceeded: @@ -337,7 +345,7 @@ def _advance_phase( if state.phase is EvolutionPhase.AWAITING_CANDIDATE: return self._candidate(request, policy, state) if state.phase is EvolutionPhase.CANDIDATE_RUNNING: - return self._complete_run(request, state) + return self._complete_run(request, policy, state) return self._advance_gate_or_wait(request, policy, state) def _advance_gate_or_wait( @@ -392,7 +400,7 @@ def _resume_intent( if state.phase is EvolutionPhase.CANDIDATE_RUNNING and ( request.run_id is not None or request.evaluated_run_receipt is not None ): - return self._complete_run(request, state) + return self._complete_run(request, policy, state) return None def _policy(self, experiment_id: str) -> ExperimentPolicySnapshot: @@ -502,7 +510,23 @@ def _candidate( state: _EvolutionState, ) -> EvolutionObservation: if request.candidate_workspace_id is not None: + if request.action not in ( + EvolutionAdvanceAction.AUTO, + EvolutionAdvanceAction.PREPARE_CANDIDATE, + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, + state.phase.value, + ) return self._prepare_candidate(request, policy, state) + if request.action not in ( + EvolutionAdvanceAction.AUTO, + EvolutionAdvanceAction.SUBMIT_CANDIDATE, + ): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.INVALID_TRANSITION, + state.phase.value, + ) return self._submit_candidate(request, state) def _prepare_candidate( @@ -527,7 +551,9 @@ def _prepare_candidate( request, state, EvolutionStopReason.MAX_ITERATIONS ) source_commit = state.accepted_commit or policy.initialization_commit - source_content_id = state.accepted_content_id or _content_identity(source_commit) + source_content_id = state.accepted_content_id or _content_identity( + source_commit + ) key = _operation_key( request.experiment_id, "candidate-prepare", state.iteration ) @@ -548,7 +574,7 @@ def _prepare_candidate( def _ensure_candidate_intent( self, request: AdvanceEvolutionInput, key: str, target: str ) -> None: - events = self._ledger.events(self._workspace_root, request.experiment_id) + events = self._events(request.experiment_id) intent = _find_intent(events, key, ExternalOperation.CANDIDATE) if intent is not None and intent.target != target: raise EvolutionControllerFailure( @@ -593,18 +619,20 @@ def _submit_candidate( def _complete_run( self, request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, state: _EvolutionState, ) -> EvolutionObservation: - run_id, receipt_id = self._run_details(request, state) + run_id, receipt_id = self._run_details(request, policy, state) return self._record_run(request, state, run_id, receipt_id) def _run_details( self, request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, state: _EvolutionState, ) -> tuple[str, str]: self._validate_run_action(request, state) - run_id, receipt_id = _run_values(request) + run_id, receipt_id = _run_values(request, policy, state) if state.run_id is not None and state.run_id != run_id: raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, run_id @@ -638,7 +666,7 @@ def _record_run( receipt_id: str, ) -> EvolutionObservation: key = _operation_key(request.experiment_id, "harbor", state.iteration) - events = self._ledger.events(self._workspace_root, request.experiment_id) + events = self._events(request.experiment_id) intent = _find_intent(events, key, ExternalOperation.HARBOR) if intent is not None and intent.target != run_id: raise EvolutionControllerFailure( @@ -709,6 +737,10 @@ def _validate_decision_identity( raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, decision.decision_id ) + if decision.decision_id != decision.recomputed_id(): + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, decision.decision_id + ) def _finish_decision( self, @@ -753,9 +785,7 @@ def _reject_candidate( ) rejects = sum( 1 - for event in self._ledger.events( - self._workspace_root, request.experiment_id - ) + for event in self._events(request.experiment_id) if event.event_type is EvolutionEventType.CANDIDATE_REJECTED ) return self._finish_rejection(request, policy, state, reasons, rejects) @@ -837,24 +867,41 @@ def _append( event_type: EvolutionEventType, payload: EvolutionEventPayload, ) -> EvolutionEvent: - return self._ledger.append( - self._workspace_root, - EvolutionEventDraft( - event_type=event_type, - experiment_id=request.experiment_id, - payload=payload, - occurred_at=request.requested_at, - causation_id=request.request_id, - correlation_id=request.request_id, - request_digest=request.digest(), - ), - ) + try: + return self._ledger.append( + self._workspace_root, + EvolutionEventDraft( + event_type=event_type, + experiment_id=request.experiment_id, + payload=payload, + occurred_at=request.requested_at, + causation_id=request.request_id, + correlation_id=request.request_id, + request_digest=request.digest(), + ), + ) + except EvolutionLedgerFailure as error: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.LEDGER_INVALID, request.experiment_id + ) from error -def _run_values(request: AdvanceEvolutionInput) -> tuple[str, str]: +def _run_values( + request: AdvanceEvolutionInput, + policy: ExperimentPolicySnapshot, + state: _EvolutionState, +) -> tuple[str, str]: receipt = request.evaluated_run_receipt if receipt is not None: - if receipt.side is not RunSide.CANDIDATE: + if ( + receipt.side is not RunSide.CANDIDATE + or receipt.policy_digest != candidate_policy_digest(policy) + or receipt.controls_digest != policy.controls_digest + or ( + state.candidate_commit is not None + and receipt.evaluated_commit != state.candidate_commit + ) + ): raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, receipt.receipt_id ) @@ -1039,7 +1086,7 @@ def _request_event( events: tuple[EvolutionEvent, ...], request_id: str ) -> EvolutionEvent | None: matches = tuple(event for event in events if event.causation_id == request_id) - return matches[0] if matches else None + return matches[-1] if matches else None def _find_intent( diff --git a/src/ofw/evolution/gate.py b/src/ofw/evolution/gate.py index e28d3bf..774a0d3 100644 --- a/src/ofw/evolution/gate.py +++ b/src/ofw/evolution/gate.py @@ -67,6 +67,12 @@ class PromotionDecision: candidate_latency_seconds: float | None canonical_json: str + def recomputed_id(self) -> str: + """Return the identity of this immutable canonical gate decision.""" + return ( + f"sha256:{hashlib.sha256(self.canonical_json.encode('utf-8')).hexdigest()}" + ) + def decide_promotion( policy: ExperimentPolicySnapshot, @@ -78,7 +84,9 @@ def decide_promotion( receipt_reasons = _receipt_reasons(policy, accepted_run, candidate_run) reasons = _ordered_reasons(identity_reasons + partition_reasons + receipt_reasons) if reasons: - return _decision(policy, accepted_run, candidate_run, PromotionStatus.INCONCLUSIVE, reasons) + return _decision( + policy, accepted_run, candidate_run, PromotionStatus.INCONCLUSIVE, reasons + ) outcome_reasons = _outcome_reasons(accepted_run, candidate_run) metric_reasons = _metric_reasons(policy, accepted_run, candidate_run) @@ -133,7 +141,9 @@ def _identity_reasons( candidate: EvaluatedRunReceipt, ) -> tuple[PromotionReason, ...]: expected_policy = candidate_policy_digest(policy) - mismatch = _authority_identity_mismatch(expected_policy, policy, accepted, candidate) + mismatch = _authority_identity_mismatch( + expected_policy, policy, accepted, candidate + ) if not mismatch: mismatch = _run_identity_mismatch(accepted, candidate) return (PromotionReason.IDENTITY_MISMATCH,) if mismatch else () @@ -164,7 +174,10 @@ def _policy_identity_mismatch( accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt, ) -> bool: - return accepted.policy_digest != expected_policy or candidate.policy_digest != expected_policy + return ( + accepted.policy_digest != expected_policy + or candidate.policy_digest != expected_policy + ) def _controls_identity_mismatch( @@ -182,7 +195,9 @@ def _side_identity_mismatch( accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt, ) -> bool: - return accepted.side is not RunSide.ACCEPTED or candidate.side is not RunSide.CANDIDATE + return ( + accepted.side is not RunSide.ACCEPTED or candidate.side is not RunSide.CANDIDATE + ) def _same_run(accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt) -> bool: @@ -229,7 +244,9 @@ def _partition_values( ) -def _has_exact_partition(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> bool: +def _has_exact_partition( + result_ids: tuple[str, ...], task_ids: tuple[str, ...] +) -> bool: return ( len(result_ids) == len(task_ids) and len(set(result_ids)) == len(result_ids) @@ -246,16 +263,23 @@ def _receipt_reasons( accepted: EvaluatedRunReceipt, candidate: EvaluatedRunReceipt, ) -> tuple[PromotionReason, ...]: - return _one_receipt_reasons(policy, accepted) + _one_receipt_reasons(policy, candidate) + return _one_receipt_reasons(policy, accepted) + _one_receipt_reasons( + policy, candidate + ) def _one_receipt_reasons( policy: ExperimentPolicySnapshot, receipt: EvaluatedRunReceipt, ) -> tuple[PromotionReason, ...]: - if receipt.receipt_id != receipt.recomputed_id() or not _task_receipt_ids_unique(receipt): + if receipt.receipt_id != receipt.recomputed_id() or not _task_receipt_ids_unique( + receipt + ): return (PromotionReason.RECEIPT_MISMATCH,) - if any(not _task_receipt_is_valid(task, policy.verifier) for task in receipt.outcome_receipts): + if any( + not _task_receipt_is_valid(task, policy.verifier) + for task in receipt.outcome_receipts + ): return (PromotionReason.RECEIPT_MISMATCH,) return () @@ -263,7 +287,9 @@ def _one_receipt_reasons( def _task_receipt_ids_unique(receipt: EvaluatedRunReceipt) -> bool: score_ids = tuple(item.score_id for item in receipt.outcome_receipts) trace_ids = tuple(item.trace_id for item in receipt.outcome_receipts) - return len(score_ids) == len(set(score_ids)) and len(trace_ids) == len(set(trace_ids)) + return len(score_ids) == len(set(score_ids)) and len(trace_ids) == len( + set(trace_ids) + ) def _task_receipt_is_valid(task: EvaluatedTaskReceipt, verifier: str) -> bool: @@ -386,7 +412,8 @@ def _has_missing_latency(receipt: EvaluatedRunReceipt) -> bool: def _exceeds_cost(receipt: EvaluatedRunReceipt, limit: float) -> bool: return any( - item.cost_usd is not None and item.cost_usd > limit for item in receipt.outcome_receipts + item.cost_usd is not None and item.cost_usd > limit + for item in receipt.outcome_receipts ) @@ -457,7 +484,9 @@ def _decision( def _passes(receipt: EvaluatedRunReceipt) -> tuple[str, ...]: return tuple( - item.task_id for item in receipt.outcome_receipts if item.verdict is VerifierVerdict.PASS + item.task_id + for item in receipt.outcome_receipts + if item.verdict is VerifierVerdict.PASS ) @@ -487,5 +516,7 @@ def _total_metric(values: tuple[float | None, ...]) -> float | None: return sum(value for value in values if value is not None) -def _ordered_reasons(reasons: tuple[PromotionReason, ...]) -> tuple[PromotionReason, ...]: +def _ordered_reasons( + reasons: tuple[PromotionReason, ...], +) -> tuple[PromotionReason, ...]: return tuple(reason for reason in PromotionReason if reason in reasons) diff --git a/src/ofw/evolution/ledger.py b/src/ofw/evolution/ledger.py index 1d0f9cc..62e7c04 100644 --- a/src/ofw/evolution/ledger.py +++ b/src/ofw/evolution/ledger.py @@ -4,12 +4,13 @@ import base64 import binascii +import fcntl import hashlib import os import re import stat from collections.abc import Iterator -from contextlib import contextmanager, suppress +from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta from enum import StrEnum @@ -25,10 +26,8 @@ from ofw.safe_file import ( SafeFileErrorCode, SafeFileFailure, - open_child_directory, open_directory_chain, read_bounded, - write_new_file, ) _DIGEST = r"sha256:[0-9a-f]{64}" @@ -217,6 +216,7 @@ class EvolutionEvent(StrictModel): causation_id: Identifier | None = None correlation_id: Identifier | None = None request_digest: Digest | None = None + payload_digest: Digest payload: EvolutionEventPayload @field_validator("occurred_at") @@ -251,6 +251,12 @@ def validate_payload_type(self) -> EvolutionEvent: return self raise ValueError("unknown event type") + @model_validator(mode="after") + def validate_payload_digest(self) -> EvolutionEvent: + if self.payload_digest != _digest(self.payload.model_dump_json()): + raise ValueError("payload_digest does not match payload") + return self + def fingerprint(self) -> str: content = self.model_dump_json(exclude={"sequence", "event_id"}) return _digest(content) @@ -287,10 +293,17 @@ def build(self, sequence: int) -> EvolutionEvent: causation_id=self.causation_id, correlation_id=self.correlation_id, request_digest=self.request_digest, + payload_digest=_digest(self.payload.model_dump_json()), payload=self.payload, ) identity = _draft_identity(self, content) - event_id = self.event_id or _digest(identity) + computed_event_id = _digest(identity) + if self.event_id is not None and self.event_id != computed_event_id: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.EVENT_CONFLICT, + self.event_id, + ) + event_id = computed_event_id return EvolutionEvent( experiment_id=content.experiment_id, sequence=content.sequence, @@ -300,6 +313,7 @@ def build(self, sequence: int) -> EvolutionEvent: causation_id=content.causation_id, correlation_id=content.correlation_id, request_digest=content.request_digest, + payload_digest=content.payload_digest, payload=content.payload, ) @@ -426,42 +440,36 @@ def _writer(control: Path) -> Iterator[int]: ("ofw", "preparations", control.name), create=True, ) as directory: + flags = os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK try: - os.mkdir(".evolution.lock", 0o700, dir_fd=directory) - except FileExistsError: + lock = os.open(".evolution.lock", flags, 0o600, dir_fd=directory) + except OSError: raise EvolutionLedgerFailure( EvolutionLedgerErrorCode.BUSY, control.name ) from None token = uuid4().hex.encode("ascii") try: - with open_child_directory( - directory, ".evolution.lock", create=False - ) as lock: - write_new_file(lock, "owner", token) - if ( - read_bounded(lock, "owner", maximum_bytes=64, subject=control.name) - != token - ): - raise EvolutionLedgerFailure( - EvolutionLedgerErrorCode.BUSY, control.name - ) - try: - yield directory - if ( - read_bounded( - lock, "owner", maximum_bytes=64, subject=control.name - ) - != token - ): - raise EvolutionLedgerFailure( - EvolutionLedgerErrorCode.BUSY, control.name - ) - finally: - with suppress(FileNotFoundError): - os.unlink("owner", dir_fd=lock) + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.BUSY, control.name + ) from None + os.ftruncate(lock, 0) + os.write(lock, token) + os.fsync(lock) + if os.pread(lock, 64, 0) != token: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.BUSY, control.name + ) + yield directory + if os.pread(lock, 64, 0) != token: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.BUSY, control.name + ) finally: - with suppress(FileNotFoundError): - os.rmdir(".evolution.lock", dir_fd=directory) + fcntl.flock(lock, fcntl.LOCK_UN) + os.close(lock) os.fsync(directory) @@ -610,6 +618,12 @@ def _append_event(directory: int, event: EvolutionEvent) -> None: event.experiment_id, event.sequence - 1, ) + if os.fstat(descriptor).st_size + len(content) > _LEDGER_LIMIT_BYTES: + raise EvolutionLedgerFailure( + EvolutionLedgerErrorCode.LEDGER_TOO_LARGE, + event.experiment_id, + event.sequence - 1, + ) view = memoryview(content) while view: view = view[os.write(descriptor, view) :] diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index f6c58fa..3c8e924 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -14,7 +14,7 @@ from mcp.server.fastmcp import FastMCP from mcp.types import ToolAnnotations -from pydantic import BaseModel, Field +from pydantic import BaseModel, BeforeValidator, Field from ofw.evaluation.failure_curation import ( FailureCurationObservation, @@ -93,7 +93,7 @@ TracePageLimit = Annotated[int, Field(strict=True, ge=1, le=50)] TaskIdentifier = Annotated[str, Field(min_length=1, max_length=256)] EvolutionExperimentIdentifier = Annotated[ - str, Field(min_length=1, max_length=80, pattern=r"[a-z0-9]+(?:-[a-z0-9]+)*") + str, Field(min_length=1, max_length=80, pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$") ] VerifierIdentifier = Annotated[str, Field(min_length=1, max_length=256)] OutcomeScore = Annotated[float, Field(strict=True, ge=0.0, le=1.0)] @@ -148,6 +148,21 @@ def _project() -> LangfuseProject: ) +def _bounded_absolute_path(value: object) -> Path: + if not isinstance(value, (str, Path)): + raise ValueError("workspace_root must be a path") + text = str(value) + if "\x00" in text or len(text.encode("utf-8")) > 1024: + raise ValueError("workspace_root must be bounded") + path = Path(value) + if not path.is_absolute(): + raise ValueError("workspace_root must be absolute") + return path + + +EvolutionWorkspaceRoot = Annotated[Path, BeforeValidator(_bounded_absolute_path)] + + def _client() -> LangfuseHttpClient: return LangfuseHttpClient(_project(), timeout_seconds=_QUERY_TIMEOUT_SECONDS) @@ -366,7 +381,8 @@ def execute_candidate( @server.tool(annotations=read_only, structured_output=True) def evolution_status( - workspace_root: Path, experiment_id: EvolutionExperimentIdentifier + workspace_root: EvolutionWorkspaceRoot, + experiment_id: EvolutionExperimentIdentifier, ) -> EvolutionObservation: """Read the replayed evolution state without appending or mutating it.""" return _evolution_controller(workspace_root).status(experiment_id) diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py index f4409bf..af8c452 100644 --- a/src/ofw/preparation/policy.py +++ b/src/ofw/preparation/policy.py @@ -241,6 +241,7 @@ def _control_directory(workspace_root: Path, experiment_id: str) -> Path: def experiment_control_directory(workspace_root: Path, experiment_id: str) -> Path: """Return the experiment directory in Git's common control area.""" + _require_experiment_id(experiment_id) result = subprocess.run( ("git", "-C", str(workspace_root), "rev-parse", "--git-common-dir"), check=False, diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index eb2f5d2..b4ab777 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -41,18 +41,12 @@ and ledger truth; never append events or perform generic transitions yourself. typed evidence; do not copy Langfuse trace payloads. 2. Use `$hypothesis-former` and `record_hypothesis`, then pass the stable hypothesis receipt to `advance_evolution`. -3. When the controller returns a candidate worktree, edit only its declared targets and use the - returned candidate worktree with `execute_candidate`; retain the candidate and evaluated run receipts. +3. Call `execute_candidate` to create the candidate worktree, edit only its declared targets, + then call it again with the identical request; retain candidate and evaluated run receipts. 4. Pass the existing `PromotionDecision` to `advance_evolution`. Accepted candidates remain `AwaitingPublication` until PR5; do not publish, merge, push, or install. -The controller returns the required next action. Use bounded trace queries and the failure -skills to retain only typed evidence; do not copy Langfuse trace payloads. Form one hypothesis -with `$hypothesis-former` and `record_hypothesis`, then pass its receipt to `advance_evolution`. -Call `execute_candidate` when requested, edit only its declared targets, and repeat the identical -request. Pass the existing `PromotionDecision` to `advance_evolution`. - ## Package boundary -Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission: -do not gate, accept, merge, publish, push, or install the candidate. +Report the hypothesis, candidate, commit, blocker, gate, and outcome receipts. Stop before +publication: do not publish, merge, push, or install the candidate. diff --git a/tests/test_evolution_controller.py b/tests/test_evolution_controller.py index 45d22f5..96f0437 100644 --- a/tests/test_evolution_controller.py +++ b/tests/test_evolution_controller.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import subprocess from datetime import UTC, datetime from pathlib import Path @@ -145,7 +146,7 @@ def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) - ) assert running.phase is EvolutionPhase.GATE_READY decision = PromotionDecision( - decision_id="sha256:" + "d" * 64, + decision_id="sha256:" + hashlib.sha256(b"{}").hexdigest(), policy_digest=candidate_policy_digest(policy()), accepted_run_id="baseline", candidate_run_id="run-1", @@ -176,6 +177,16 @@ def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) - with pytest.raises(EvolutionControllerFailure) as raised: controller.advance(_request(root, "r7", release_id="release-1")) assert raised.value.code is EvolutionControllerErrorCode.PUBLICATION_REQUIRED + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request( + root, + "r8", + action=EvolutionAdvanceAction.BLOCK, + blocker_reason="publication", + ) + ) + assert raised.value.code is EvolutionControllerErrorCode.PUBLICATION_REQUIRED assert ( controller.status("experiment-one").phase is EvolutionPhase.AWAITING_PUBLICATION ) @@ -198,6 +209,8 @@ def test_explicit_stop_is_typed_and_terminal(tmp_path: Path) -> None: def test_input_and_workspace_boundaries_are_strict(tmp_path: Path) -> None: + with pytest.raises(EvolutionControllerFailure): + EvolutionController(workspace_root=Path("relative")) with pytest.raises(ValidationError): AdvanceEvolutionInput( workspace_root=Path("relative"), @@ -323,7 +336,7 @@ def test_max_iterations_and_no_improvement_stops(tmp_path: Path) -> None: _request(root, "r5", run_id="run-1", candidate_receipt_id="sha256:" + "c" * 64) ) decision = PromotionDecision( - decision_id="sha256:" + "d" * 64, + decision_id="sha256:" + hashlib.sha256(b"{}").hexdigest(), policy_digest=candidate_policy_digest( policy(max_iterations=1, no_improvement_limit=1) ), diff --git a/tests/test_evolution_ledger.py b/tests/test_evolution_ledger.py index b0d1931..3fc5a84 100644 --- a/tests/test_evolution_ledger.py +++ b/tests/test_evolution_ledger.py @@ -87,6 +87,7 @@ def test_empty_log_and_invalid_draft_inputs_are_handled(tmp_path: Path) -> None: event_id="sha256:" + "a" * 64, event_type=EvolutionEventType.EVOLUTION_STARTED, occurred_at=datetime(2026, 9, 3), + payload_digest="sha256:" + "a" * 64, payload=EvolutionStarted(policy_digest="sha256:" + "a" * 64), ) @@ -160,6 +161,7 @@ def test_invalid_workspace_and_payload_fail_closed(tmp_path: Path) -> None: event_id="sha256:" + "a" * 64, event_type=EvolutionEventType.EVOLUTION_STARTED, occurred_at=datetime(2026, 9, 3, tzinfo=UTC), + payload_digest="sha256:" + "a" * 64, payload=HypothesisLinked( hypothesis_id="sha256:" + "a" * 64, source_commit="a" * 40, @@ -170,8 +172,8 @@ def test_invalid_workspace_and_payload_fail_closed(tmp_path: Path) -> None: def test_writer_owner_lock_is_exclusive(tmp_path: Path) -> None: root = _repo(tmp_path) ledger = FileEvolutionLedger() - ledger.append(root, _draft()) lock = root / ".git" / "ofw" / "preparations" / "experiment-one" / ".evolution.lock" + lock.parent.mkdir(parents=True) lock.mkdir() with pytest.raises(EvolutionLedgerFailure) as raised: ledger.append(root, _draft("experiment-one")) diff --git a/tests/test_program_templates.py b/tests/test_program_templates.py index 8a6621a..8dddddf 100644 --- a/tests/test_program_templates.py +++ b/tests/test_program_templates.py @@ -8,7 +8,9 @@ @pytest.mark.parametrize("name", ("base.md", "itsm.md")) def test_packaged_program_template_matches_plugin_asset(name: str) -> None: - plugin_path = Path(__file__).parents[1] / "plugins/openflywheel/program_templates" / name + plugin_path = ( + Path(__file__).parents[1] / "plugins/openflywheel/program_templates" / name + ) packaged = files("ofw.preparation.templates").joinpath(name).read_bytes() assert packaged == plugin_path.read_bytes() @@ -31,7 +33,11 @@ def test_packaged_program_template_matches_plugin_asset(name: str) -> None: def test_itsm_program_routes_failure_mining_to_local_workspace_artifacts( required_instruction: str, ) -> None: - content = files("ofw.preparation.templates").joinpath("itsm.md").read_text(encoding="utf-8") + content = ( + files("ofw.preparation.templates") + .joinpath("itsm.md") + .read_text(encoding="utf-8") + ) assert required_instruction in content @@ -48,13 +54,20 @@ def test_itsm_program_routes_failure_mining_to_local_workspace_artifacts( def test_itsm_program_routes_recorded_diagnoses_to_bounded_pattern_mining( required_instruction: str, ) -> None: - content = files("ofw.preparation.templates").joinpath("itsm.md").read_text(encoding="utf-8") + content = ( + files("ofw.preparation.templates") + .joinpath("itsm.md") + .read_text(encoding="utf-8") + ) assert required_instruction in content def test_failure_pattern_miner_skill_is_packaged() -> None: - skill = Path(__file__).parents[1] / "plugins/openflywheel/skills/failure-pattern-miner/SKILL.md" + skill = ( + Path(__file__).parents[1] + / "plugins/openflywheel/skills/failure-pattern-miner/SKILL.md" + ) assert skill.is_file() assert "mine_failure_patterns" in skill.read_text(encoding="utf-8") @@ -63,19 +76,27 @@ def test_failure_pattern_miner_skill_is_packaged() -> None: def test_program_routes_hypothesis_receipt_into_candidate_execution() -> None: root = Path(__file__).parents[1] skill = root / "plugins/openflywheel/skills/hypothesis-former/SKILL.md" - program = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") + program = ( + files("ofw.preparation.templates") + .joinpath("base.md") + .read_text(encoding="utf-8") + ) assert skill.is_file() assert "record_hypothesis" in skill.read_text(encoding="utf-8") assert "$hypothesis-former" in program assert "stable hypothesis receipt" in program assert "execute_candidate" in program - assert "returned candidate worktree" in program - assert "stop before admission" in program + assert "identical request" in program + assert "Stop before" in program def test_base_program_stops_after_repeated_managed_mcp_timeout() -> None: - content = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") + content = ( + files("ofw.preparation.templates") + .joinpath("base.md") + .read_text(encoding="utf-8") + ) content = " ".join(content.split()) assert "unknown operation status" in content From 0b786bcd127bf033430476cb50febc5793be3387 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:23:08 +0530 Subject: [PATCH 07/11] fix: preserve evolution replay state --- src/ofw/evolution/controller.py | 23 +++++++++++++++++++++++ src/ofw/evolution/ledger.py | 8 ++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py index eb7d051..93b7bbf 100644 --- a/src/ofw/evolution/controller.py +++ b/src/ofw/evolution/controller.py @@ -979,6 +979,14 @@ def _apply_release( stop_reason=( EvolutionStopReason.QUALITY_TARGET if target_reached else None ), + hypothesis_id=None, + candidate_workspace_id=None, + candidate_id=None, + candidate_commit=None, + run_id=None, + candidate_receipt_id=None, + decision_id=None, + gate_status=None, ) return replace( state, @@ -987,6 +995,14 @@ def _apply_release( accepted_commit=payload.content_commit or state.accepted_commit, accepted_content_id=payload.content_id or state.accepted_content_id, stop_reason=None, + hypothesis_id=None, + candidate_workspace_id=None, + candidate_id=None, + candidate_commit=None, + run_id=None, + candidate_receipt_id=None, + decision_id=None, + gate_status=None, ) @@ -998,6 +1014,13 @@ def _apply_candidate_preparation( state, phase=EvolutionPhase.AWAITING_CANDIDATE, hypothesis_id=payload.hypothesis_id, + candidate_workspace_id=None, + candidate_id=None, + candidate_commit=None, + run_id=None, + candidate_receipt_id=None, + decision_id=None, + gate_status=None, ) return replace( state, diff --git a/src/ofw/evolution/ledger.py b/src/ofw/evolution/ledger.py index 62e7c04..caa55bc 100644 --- a/src/ofw/evolution/ledger.py +++ b/src/ofw/evolution/ledger.py @@ -216,7 +216,7 @@ class EvolutionEvent(StrictModel): causation_id: Identifier | None = None correlation_id: Identifier | None = None request_digest: Digest | None = None - payload_digest: Digest + payload_digest: Digest | None = None payload: EvolutionEventPayload @field_validator("occurred_at") @@ -253,7 +253,9 @@ def validate_payload_type(self) -> EvolutionEvent: @model_validator(mode="after") def validate_payload_digest(self) -> EvolutionEvent: - if self.payload_digest != _digest(self.payload.model_dump_json()): + if self.payload_digest is not None and self.payload_digest != _digest( + self.payload.model_dump_json() + ): raise ValueError("payload_digest does not match payload") return self @@ -556,6 +558,8 @@ def _validate_event_order( def _validate_event_identity( event: EvolutionEvent, experiment_id: str, last: int ) -> None: + if event.payload_digest is None: + return identity = _event_identity(event) if event.event_id != _digest(identity): raise EvolutionLedgerFailure( From c719e050fbe4ca1be6c646eba437a11825c70f1d Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:09:33 +0530 Subject: [PATCH 08/11] fix: preserve latest PR3 namespace contracts --- src/ofw/__init__.py | 1 - tests/support_policy.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index c36094b..d30da4e 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -50,7 +50,6 @@ ) from ofw.evolution import ( AdvanceEvolutionInput, - CandidateBlocker, CandidateBlockerCode, CandidateErrorCode, CandidateExecutionInput, diff --git a/tests/support_policy.py b/tests/support_policy.py index a08866a..a121b7b 100644 --- a/tests/support_policy.py +++ b/tests/support_policy.py @@ -62,6 +62,8 @@ def load(self, workspace_root: Path, hypothesis_id: str) -> HarnessHypothesis: curation_id="00000000-0000-0000-0000-000000000001", curation_group_id="00000000-0000-0000-0000-000000000002", patterns=(), + predicted_task_ids=("task-1",), + at_risk_task_ids=(), statement="statement", rationale="rationale", target=HarnessChangeTarget(ComponentKind.SKILL, (Path("PROGRAM.md"),)), From cd5e2316fa2fcedae6f76afa7b286768d63dd945 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:20:25 +0530 Subject: [PATCH 09/11] fix: authenticate rebased evolution gate receipts --- .../openflywheel/program_templates/base.md | 2 +- src/ofw/evolution/controller.py | 17 +- src/ofw/preparation/templates/base.md | 2 +- tests/test_evolution_controller.py | 152 +++++++++++------- 4 files changed, 116 insertions(+), 57 deletions(-) diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index b4ab777..93d4851 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -5,7 +5,7 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission Call `evolution_status`, then `advance_evolution` for exactly one next action under the -canonical experiment policy. Stop before admission. +canonical experiment policy. Stop before publication. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py index 93b7bbf..efaac57 100644 --- a/src/ofw/evolution/controller.py +++ b/src/ofw/evolution/controller.py @@ -13,7 +13,7 @@ from ofw.evaluation.outcome import EvaluatedRunReceipt, RunSide from ofw.evolution.candidate import candidate_policy_digest -from ofw.evolution.gate import PromotionDecision, PromotionStatus +from ofw.evolution.gate import PromotionDecision, PromotionStatus, decide_promotion from ofw.evolution.hypothesis import HarnessHypothesis, HypothesisFailure from ofw.evolution.hypothesis_repository import FileHypothesisRepository from ofw.evolution.ledger import ( @@ -137,6 +137,7 @@ class AdvanceEvolutionInput(StrictModel): run_id: str | None = Field(default=None, max_length=256, pattern=_IDENTIFIER) candidate_receipt_id: str | None = Field(default=None, pattern=_DIGEST) evaluated_run_receipt: EvaluatedRunReceipt | None = None + accepted_run_receipt: EvaluatedRunReceipt | None = None promotion_decision: PromotionDecision | None = None release_id: str | None = Field(default=None, max_length=256, pattern=_IDENTIFIER) stop_reason: EvolutionStopReason | None = None @@ -721,6 +722,20 @@ def _validate_decision( EvolutionControllerErrorCode.MISSING_INPUT, "promotion_decision" ) self._validate_decision_identity(decision, policy, state) + candidate = request.evaluated_run_receipt + accepted = request.accepted_run_receipt + if candidate is None or accepted is None: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.MISSING_INPUT, "gate_receipts" + ) + if candidate.receipt_id != state.candidate_receipt_id: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, candidate.receipt_id + ) + if decide_promotion(policy, accepted, candidate) != decision: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, decision.decision_id + ) return decision, tuple(reason.value for reason in decision.reasons) def _validate_decision_identity( diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index b4ab777..93d4851 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -5,7 +5,7 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission Call `evolution_status`, then `advance_evolution` for exactly one next action under the -canonical experiment policy. Stop before admission. +canonical experiment policy. Stop before publication. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an diff --git a/tests/test_evolution_controller.py b/tests/test_evolution_controller.py index 96f0437..d052be6 100644 --- a/tests/test_evolution_controller.py +++ b/tests/test_evolution_controller.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import subprocess from datetime import UTC, datetime from pathlib import Path @@ -8,6 +7,12 @@ import pytest from pydantic import ValidationError +from ofw.evaluation.outcome import ( + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + RunSide, + VerifierVerdict, +) from ofw.evolution.candidate import candidate_policy_digest from ofw.evolution.controller import ( AdvanceEvolutionInput, @@ -19,7 +24,7 @@ EvolutionStatus, EvolutionStopReason, ) -from ofw.evolution.gate import PromotionDecision, PromotionReason, PromotionStatus +from ofw.evolution.gate import PromotionDecision, decide_promotion from ofw.evolution.ledger import ( CandidatePrepared, EvolutionEvent, @@ -81,6 +86,8 @@ def _request( candidate_commit: str | None = None, run_id: str | None = None, candidate_receipt_id: str | None = None, + evaluated_run_receipt: EvaluatedRunReceipt | None = None, + accepted_run_receipt: EvaluatedRunReceipt | None = None, promotion_decision: PromotionDecision | None = None, release_id: str | None = None, stop_reason: EvolutionStopReason | None = None, @@ -103,6 +110,8 @@ def _request( candidate_commit=candidate_commit, run_id=run_id, candidate_receipt_id=candidate_receipt_id, + evaluated_run_receipt=evaluated_run_receipt, + accepted_run_receipt=accepted_run_receipt, promotion_decision=promotion_decision, release_id=release_id, stop_reason=stop_reason, @@ -112,6 +121,38 @@ def _request( ) +def _receipt( + run_id: str, + side: RunSide, + commit: str, + verdict: VerifierVerdict, + experiment_policy: ExperimentPolicySnapshot | None = None, +) -> EvaluatedRunReceipt: + authority = experiment_policy or policy() + score = 1.0 if verdict is VerifierVerdict.PASS else 0.0 + task = EvaluatedTaskReceipt( + task_id="task-1", + trace_id=f"trace-{run_id}", + score_id=f"score-{run_id}", + verdict=verdict, + verifier_id="verifier", + normalized_score=score, + cost_usd=0.1, + latency_seconds=1.0, + ) + return EvaluatedRunReceipt.build( + run_id=run_id, + side=side, + policy_digest=candidate_policy_digest(authority), + controls_digest=authority.controls_digest, + evaluated_commit=commit, + evaluated_tree=commit, + task_ids=("task-1",), + outcome_receipts=(task,), + blockers=(), + ) + + def test_controller_advances_one_deterministic_step_and_retries_idempotently( tmp_path: Path, ) -> None: @@ -138,42 +179,44 @@ def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) - controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) controller.advance( _request( - root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="c" * 40 ) ) + accepted_receipt = _receipt( + "baseline", RunSide.ACCEPTED, "a" * 40, VerifierVerdict.FAIL + ) + candidate_receipt = _receipt( + "run-1", RunSide.CANDIDATE, "c" * 40, VerifierVerdict.PASS + ) running = controller.advance( - _request(root, "r5", run_id="run-1", candidate_receipt_id="sha256:" + "c" * 64) + _request( + root, + "r5", + run_id="run-1", + candidate_receipt_id=candidate_receipt.receipt_id, + ) ) assert running.phase is EvolutionPhase.GATE_READY - decision = PromotionDecision( - decision_id="sha256:" + hashlib.sha256(b"{}").hexdigest(), - policy_digest=candidate_policy_digest(policy()), - accepted_run_id="baseline", - candidate_run_id="run-1", - status=PromotionStatus.ACCEPT, - reasons=(PromotionReason.IMPROVEMENT,), - task_ids=("task-1",), - accepted_passes=(), - candidate_passes=("task-1",), - accepted_quality=0.0, - candidate_quality=1.0, - accepted_cost_usd=None, - candidate_cost_usd=None, - accepted_latency_seconds=None, - candidate_latency_seconds=None, - canonical_json="{}", - ) - accepted = controller.advance(_request(root, "r6", promotion_decision=decision)) + decision = decide_promotion(policy(), accepted_receipt, candidate_receipt) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request(root, "r6-missing-receipts", promotion_decision=decision) + ) + assert raised.value.code is EvolutionControllerErrorCode.MISSING_INPUT + accepted = controller.advance( + _request( + root, + "r6", + promotion_decision=decision, + evaluated_run_receipt=candidate_receipt, + accepted_run_receipt=accepted_receipt, + ) + ) assert accepted.phase is EvolutionPhase.AWAITING_PUBLICATION assert (accepted.accepted_commit, accepted.accepted_content_id) == ( - "a" * 40, + "c" * 40, "sha256:" + "b" * 64, ) - from ofw.evolution.controller import ( - EvolutionControllerErrorCode, - EvolutionControllerFailure, - ) - with pytest.raises(EvolutionControllerFailure) as raised: controller.advance(_request(root, "r7", release_id="release-1")) assert raised.value.code is EvolutionControllerErrorCode.PUBLICATION_REQUIRED @@ -285,7 +328,7 @@ def test_block_retry_and_conflicting_external_targets(tmp_path: Path) -> None: controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) controller.advance( _request( - root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="c" * 40 ) ) blocked = controller.advance( @@ -309,7 +352,7 @@ def test_block_retry_and_conflicting_external_targets(tmp_path: Path) -> None: controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) controller.advance( _request( - root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="c" * 40 ) ) with pytest.raises(EvolutionControllerFailure) as raised: @@ -329,33 +372,34 @@ def test_max_iterations_and_no_improvement_stops(tmp_path: Path) -> None: controller.advance(_request(root, "r3", candidate_workspace_id="workspace-1")) controller.advance( _request( - root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="a" * 40 + root, "r4", candidate_id="sha256:" + "b" * 64, candidate_commit="c" * 40 ) ) + authority = policy(max_iterations=1, no_improvement_limit=1) + accepted_receipt = _receipt( + "baseline", RunSide.ACCEPTED, "a" * 40, VerifierVerdict.PASS, authority + ) + candidate_receipt = _receipt( + "run-1", RunSide.CANDIDATE, "c" * 40, VerifierVerdict.PASS, authority + ) controller.advance( - _request(root, "r5", run_id="run-1", candidate_receipt_id="sha256:" + "c" * 64) + _request( + root, + "r5", + run_id="run-1", + candidate_receipt_id=candidate_receipt.receipt_id, + ) + ) + decision = decide_promotion(authority, accepted_receipt, candidate_receipt) + stopped = controller.advance( + _request( + root, + "r6", + promotion_decision=decision, + evaluated_run_receipt=candidate_receipt, + accepted_run_receipt=accepted_receipt, + ) ) - decision = PromotionDecision( - decision_id="sha256:" + hashlib.sha256(b"{}").hexdigest(), - policy_digest=candidate_policy_digest( - policy(max_iterations=1, no_improvement_limit=1) - ), - accepted_run_id="baseline", - candidate_run_id="run-1", - status=PromotionStatus.REJECT, - reasons=(PromotionReason.NO_IMPROVEMENT,), - task_ids=("task-1",), - accepted_passes=("task-1",), - candidate_passes=("task-1",), - accepted_quality=1.0, - candidate_quality=1.0, - accepted_cost_usd=None, - candidate_cost_usd=None, - accepted_latency_seconds=None, - candidate_latency_seconds=None, - canonical_json="{}", - ) - stopped = controller.advance(_request(root, "r6", promotion_decision=decision)) assert stopped.stop_reason is EvolutionStopReason.MAX_ITERATIONS From 372da81765a6e78935e3b5d2814160276531c6c7 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:32:52 +0530 Subject: [PATCH 10/11] fix: bind evolution receipts to accepted commits --- src/ofw/evolution/controller.py | 8 ++++++-- tests/test_evolution_controller.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py index efaac57..4dab8c7 100644 --- a/src/ofw/evolution/controller.py +++ b/src/ofw/evolution/controller.py @@ -728,6 +728,10 @@ def _validate_decision( raise EvolutionControllerFailure( EvolutionControllerErrorCode.MISSING_INPUT, "gate_receipts" ) + if accepted.evaluated_commit != state.accepted_commit: + raise EvolutionControllerFailure( + EvolutionControllerErrorCode.STALE_RECEIPT, accepted.receipt_id + ) if candidate.receipt_id != state.candidate_receipt_id: raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, candidate.receipt_id @@ -913,8 +917,8 @@ def _run_values( or receipt.policy_digest != candidate_policy_digest(policy) or receipt.controls_digest != policy.controls_digest or ( - state.candidate_commit is not None - and receipt.evaluated_commit != state.candidate_commit + state.candidate_commit is None + or receipt.evaluated_commit != state.candidate_commit ) ): raise EvolutionControllerFailure( diff --git a/tests/test_evolution_controller.py b/tests/test_evolution_controller.py index d052be6..3392b89 100644 --- a/tests/test_evolution_controller.py +++ b/tests/test_evolution_controller.py @@ -188,6 +188,20 @@ def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) - candidate_receipt = _receipt( "run-1", RunSide.CANDIDATE, "c" * 40, VerifierVerdict.PASS ) + wrong_candidate = _receipt( + "run-wrong", RunSide.CANDIDATE, "a" * 40, VerifierVerdict.PASS + ) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request( + root, + "r5-wrong-candidate", + run_id=wrong_candidate.run_id, + evaluated_run_receipt=wrong_candidate, + candidate_receipt_id=wrong_candidate.receipt_id, + ) + ) + assert raised.value.code is EvolutionControllerErrorCode.STALE_RECEIPT running = controller.advance( _request( root, @@ -198,6 +212,20 @@ def test_accepted_candidate_is_stuck_until_publication_package(tmp_path: Path) - ) assert running.phase is EvolutionPhase.GATE_READY decision = decide_promotion(policy(), accepted_receipt, candidate_receipt) + wrong_accepted = _receipt( + "baseline", RunSide.ACCEPTED, "c" * 40, VerifierVerdict.FAIL + ) + with pytest.raises(EvolutionControllerFailure) as raised: + controller.advance( + _request( + root, + "r6-wrong-accepted", + promotion_decision=decision, + evaluated_run_receipt=candidate_receipt, + accepted_run_receipt=wrong_accepted, + ) + ) + assert raised.value.code is EvolutionControllerErrorCode.STALE_RECEIPT with pytest.raises(EvolutionControllerFailure) as raised: controller.advance( _request(root, "r6-missing-receipts", promotion_decision=decision) From 8fde9ef0deeeda99f615354c99cbaff46ea23659 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:48:14 +0530 Subject: [PATCH 11/11] fix: fall back to persisted baseline commit --- src/ofw/evolution/controller.py | 4 +- tests/test_evolution_controller.py | 78 +++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/src/ofw/evolution/controller.py b/src/ofw/evolution/controller.py index 4dab8c7..22c6c6e 100644 --- a/src/ofw/evolution/controller.py +++ b/src/ofw/evolution/controller.py @@ -728,7 +728,9 @@ def _validate_decision( raise EvolutionControllerFailure( EvolutionControllerErrorCode.MISSING_INPUT, "gate_receipts" ) - if accepted.evaluated_commit != state.accepted_commit: + if accepted.evaluated_commit != ( + state.accepted_commit or policy.initialization_commit + ): raise EvolutionControllerFailure( EvolutionControllerErrorCode.STALE_RECEIPT, accepted.receipt_id ) diff --git a/tests/test_evolution_controller.py b/tests/test_evolution_controller.py index 3392b89..c659d97 100644 --- a/tests/test_evolution_controller.py +++ b/tests/test_evolution_controller.py @@ -27,21 +27,26 @@ from ofw.evolution.gate import PromotionDecision, decide_promotion from ofw.evolution.ledger import ( CandidatePrepared, + CandidateSubmitted, EvolutionEvent, EvolutionEventDraft, + EvolutionEventPayload, EvolutionEventType, EvolutionLedgerErrorCode, EvolutionLedgerFailure, + EvolutionStarted, FileEvolutionLedger, + HypothesisLinked, ReleasePublished, ReleaseRolledBack, + RunCompleted, ) from ofw.preparation.policy import ( ExperimentPolicyErrorCode, ExperimentPolicyFailure, ExperimentPolicySnapshot, ) -from tests.support_policy import PolicyRepository, policy +from tests.support_policy import HypothesisRepository, PolicyRepository, policy def _repo(tmp_path: Path) -> Path: @@ -500,6 +505,77 @@ def test_conflicting_controller_retry_is_rejected(tmp_path: Path) -> None: assert raised.value.code is EvolutionControllerErrorCode.REQUEST_CONFLICT +def test_legacy_started_event_uses_policy_baseline_for_gate(tmp_path: Path) -> None: + root = _repo(tmp_path) + authority = policy() + ledger = FileEvolutionLedger() + timestamp = datetime(2026, 9, 3, tzinfo=UTC) + + def append( + event_type: EvolutionEventType, + payload: EvolutionEventPayload, + name: str, + ) -> None: + ledger.append( + root, + EvolutionEventDraft( + event_type=event_type, + experiment_id="experiment-one", + payload=payload, + occurred_at=timestamp, + causation_id=name, + correlation_id=name, + ), + ) + + append( + EvolutionEventType.EVOLUTION_STARTED, + EvolutionStarted(policy_digest=candidate_policy_digest(authority)), + "start", + ) + append( + EvolutionEventType.HYPOTHESIS_LINKED, + HypothesisLinked(hypothesis_id="sha256:" + "a" * 64, source_commit="a" * 40), + "hypothesis", + ) + append( + EvolutionEventType.CANDIDATE_PREPARED, + CandidatePrepared(iteration=1, candidate_workspace_id="workspace-1"), + "prepare", + ) + append( + EvolutionEventType.CANDIDATE_SUBMITTED, + CandidateSubmitted( + candidate_id="sha256:" + "b" * 64, candidate_commit="c" * 40 + ), + "submit", + ) + candidate = _receipt("run-1", RunSide.CANDIDATE, "c" * 40, VerifierVerdict.PASS) + append( + EvolutionEventType.RUN_COMPLETED, + RunCompleted(run_id="run-1", receipt_id=candidate.receipt_id), + "complete", + ) + accepted = _receipt("baseline", RunSide.ACCEPTED, "a" * 40, VerifierVerdict.FAIL) + decision = decide_promotion(authority, accepted, candidate) + controller = EvolutionController( + workspace_root=root, + ledger=ledger, + policy_repository=PolicyRepository(authority), + hypothesis_repository=HypothesisRepository(), + ) + observation = controller.advance( + _request( + root, + "decide", + promotion_decision=decision, + evaluated_run_receipt=candidate, + accepted_run_receipt=accepted, + ) + ) + assert observation.phase is EvolutionPhase.AWAITING_PUBLICATION + + def test_illegal_action_and_stopped_state_are_typed(tmp_path: Path) -> None: root = _repo(tmp_path) controller = _controller(root)