feat: execute isolated candidates

plugins/openflywheel/.codex-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "openflywheel",
   "version": "0.8.0",
-  "description": "Prepare ITSM-bench workspaces and record evidence-backed harness hypotheses.",
+  "description": "Prepare ITSM-bench workspaces, record hypotheses, and execute isolated candidates.",
   "author": {
     "name": "OpenFlyWheel"
   },
@@ -10,14 +10,14 @@
   "skills": "./skills/",
   "interface": {
     "displayName": "OpenFlyWheel",
-    "shortDescription": "Record evidence-backed harness hypotheses",
-    "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, and persist one evidence-backed hypothesis before candidate editing without copying trace payloads.",
+    "shortDescription": "Execute isolated harness candidates",
+    "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, persist one evidence-backed hypothesis, execute one isolated candidate, and stop before admission without copying trace payloads.",
     "developerName": "OpenFlyWheel",
     "category": "Productivity",
     "capabilities": ["Read", "Write"],
     "defaultPrompt": [
       "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.",
-      "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, and stop before candidate editing."
+      "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, execute one isolated candidate, and stop before admission."
     ]
   },
   "mcpServers": "./.mcp.json"

plugins/openflywheel/.mcp.json

@@ -4,7 +4,7 @@
       "command": "uvx",
       "args": [
         "--from",
-        "git+https://github.com/divo12/OpenFlyWheel.git@fe5d0972ef4733553454814e96910ae51ab6a5a3",
+        "git+https://github.com/divo12/OpenFlyWheel.git@9041db3c08a89df0fe9f8f2476a303b46dd2812a",
         "--with",
         "mcp>=1.13,<2",
         "openflywheel-mcp"

plugins/openflywheel/program_templates/base.md

@@ -4,8 +4,8 @@ This file is generated by `prepare_workspace`. Do not edit it directly.
 
 ## Mission
 
-Record one evidence-backed hypothesis for the connected agent harness under the canonical
-experiment policy, then stop before candidate editing.
+Record one evidence-backed hypothesis, execute its isolated candidate under the canonical
+experiment policy, then stop before admission.
 
 The baseline has already been recorded. Begin at step 2; do not rerun the unchanged
 baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an
@@ -27,9 +27,9 @@ existing terminal Harbor job was adopted).
 
 ## Editable and frozen surfaces
 
-Target only exact paths allowed by the canonical experiment policy. Do not edit them in this
-program. Never target the benchmark, held-out tasks, verifier, model, reasoning budget,
-observability identity, or this program.
+Target only exact paths allowed by the canonical experiment policy. Edit them only in the
+candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks,
+verifier, model, reasoning budget, observability identity, or this program.
 
 Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control
 flow changes unless the evidence requires the combination.
@@ -51,10 +51,20 @@ between materially different changes.
 
 Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported
 pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call
-`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing.
-Candidate editing requires a later package and must not begin in this prepared program.
+`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution.
+
+### 4. Execute one candidate
+
+Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling
+candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated
+worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the
+returned candidate worktree, then call `execute_candidate` again with the identical request.
+
+Poll identical requests while the candidate is running. Retain its candidate ID, Git commit,
+trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally,
+change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree.
 
 ## Package boundary
 
-Report the hypothesis receipt and its exact evidence and target paths. Do not edit, commit,
-run, gate, publish, or install a candidate; those capabilities are not part of this program.
+Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission:
+do not gate, accept, merge, publish, push, or install the candidate.

src/ofw/__init__.py

@@ -13,6 +13,9 @@
 from ofw.contracts import ComponentKind, Sha256Digest
 from ofw.evaluation import (
     DeferredFailure,
+    EvaluatedRunBlocker,
+    EvaluatedRunReceipt,
+    EvaluatedTaskReceipt,
     EvidenceReference,
     FailureCuration,
     FailureCurationErrorCode,
@@ -39,12 +42,21 @@
     OutcomeScoreSubmission,
     OutcomeStoreObservation,
     OutcomeStoreStatus,
+    RunSide,
     TaskId,
     VerifierId,
     VerifierResult,
     VerifierVerdict,
 )
 from ofw.evolution import (
+    CandidateBlockerCode,
+    CandidateErrorCode,
+    CandidateExecutionInput,
+    CandidateExecutionObservation,
+    CandidateFailure,
+    CandidateId,
+    CandidatePhase,
+    CandidateStatus,
     FailurePatternReference,
     FailurePatternReferenceInput,
     HarnessChangeTarget,
@@ -76,11 +88,22 @@
 )
 
 __all__ = [
+    "CandidateBlockerCode",
+    "CandidateErrorCode",
+    "CandidateExecutionInput",
+    "CandidateExecutionObservation",
+    "CandidateFailure",
+    "CandidateId",
+    "CandidatePhase",
+    "CandidateStatus",
     "CollectionError",
     "CollectionErrorCode",
     "ComponentKind",
     "DeferredFailure",
     "EvidenceReference",
+    "EvaluatedRunBlocker",
+    "EvaluatedRunReceipt",
+    "EvaluatedTaskReceipt",
     "ExperimentPolicyErrorCode",
     "ExperimentPolicyFailure",
     "ExperimentPolicySnapshot",
@@ -123,6 +146,7 @@
     "OutcomeScoreSubmission",
     "OutcomeStoreObservation",
     "OutcomeStoreStatus",
+    "RunSide",
     "PreparationErrorCode",
     "PreparationPhase",
     "PreparationStatus",

src/ofw/evaluation/__init__.py

@@ -32,10 +32,14 @@
     OutcomeStoreStatus,
 )
 from ofw.evaluation.outcome import (
+    EvaluatedRunBlocker,
+    EvaluatedRunReceipt,
+    EvaluatedTaskReceipt,
     EvidenceReference,
     OutcomeErrorCode,
     OutcomeEvaluation,
     OutcomeEvaluationError,
+    RunSide,
     TaskId,
     VerifierId,
     VerifierResult,
@@ -45,6 +49,9 @@
 __all__ = [
     "DeferredFailure",
     "EvidenceReference",
+    "EvaluatedRunBlocker",
+    "EvaluatedRunReceipt",
+    "EvaluatedTaskReceipt",
     "FailureCuration",
     "FailureCurationErrorCode",
     "FailureCurationFailure",
@@ -67,6 +74,7 @@
     "OutcomeErrorCode",
     "OutcomeEvaluation",
     "OutcomeEvaluationError",
+    "RunSide",
     "OutcomeScoreSubmission",
     "OutcomeStoreObservation",
     "OutcomeStoreStatus",

src/ofw/evaluation/outcome.py

@@ -2,18 +2,27 @@
 
 from __future__ import annotations
 
+import hashlib
 import math
 import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import StrEnum
 
+from pydantic import BaseModel, ConfigDict, Field, StrictStr, model_validator
+
 from ofw.observability.langfuse.domain import TraceId
 
 _IDENTIFIER_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]*")
 _IDENTIFIER_LIMIT = 256
 _EVIDENCE_LIMIT = 10
 _EVIDENCE_VALUE_LIMIT = 1024
+_RUN_ID_LIMIT = 256
+_COMMIT_PATTERN = r"^[0-9a-f]{40}$"
+_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$"
+_RUN_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
+_RUN_METRIC_LIMIT = 172800.0
+_COST_LIMIT = 1_000_000.0
 
 
 class OutcomeErrorCode(StrEnum):
@@ -43,6 +52,168 @@ class VerifierVerdict(StrEnum):
     ERROR = "error"
 
 
+class RunSide(StrEnum):
+    ACCEPTED = "accepted"
+    CANDIDATE = "candidate"
+
+
+class _ReceiptModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+
+class EvaluatedTaskReceipt(_ReceiptModel):
+    task_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+    trace_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+    score_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+    verdict: VerifierVerdict
+    verifier_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+    normalized_score: float | None = None
+    cost_usd: float | None = None
+    latency_seconds: float | None = None
+
+    @model_validator(mode="after")
+    def validate_metrics(self) -> EvaluatedTaskReceipt:
+        _validate_run_metric(self.normalized_score, 0.0, 1.0, "normalized_score")
+        _validate_run_metric(self.cost_usd, 0.0, _COST_LIMIT, "cost_usd")
+        _validate_run_metric(
+            self.latency_seconds,
+            0.0,
+            _RUN_METRIC_LIMIT,
+            "latency_seconds",
+        )
+        _validate_verdict_score(self.verdict, self.normalized_score)
+        return self
+
+
+class EvaluatedRunBlocker(_ReceiptModel):
+    task_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+    code: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+    subject: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT)
+
+
+class EvaluatedRunReceipt(_ReceiptModel):
+    receipt_id: StrictStr = Field(pattern=_DIGEST_PATTERN)
+    run_id: StrictStr = Field(
+        min_length=1,
+        max_length=_RUN_ID_LIMIT,
+        pattern=_RUN_ID_PATTERN,
+    )
+    side: RunSide
+    policy_digest: StrictStr = Field(pattern=_DIGEST_PATTERN)
+    controls_digest: StrictStr = Field(pattern=_DIGEST_PATTERN)
+    evaluated_commit: StrictStr = Field(pattern=_COMMIT_PATTERN)
+    evaluated_tree: StrictStr = Field(pattern=_COMMIT_PATTERN)
+    task_ids: tuple[StrictStr, ...] = Field(min_length=1, max_length=500)
+    outcome_receipts: tuple[EvaluatedTaskReceipt, ...] = Field(max_length=500)
+    blockers: tuple[EvaluatedRunBlocker, ...] = Field(max_length=500)
+
+    @classmethod
+    def build(
+        cls,
+        *,
+        run_id: str,
+        side: RunSide,
+        policy_digest: str,
+        controls_digest: str,
+        evaluated_commit: str,
+        evaluated_tree: str,
+        task_ids: tuple[str, ...],
+        outcome_receipts: tuple[EvaluatedTaskReceipt, ...],
+        blockers: tuple[EvaluatedRunBlocker, ...],
+    ) -> EvaluatedRunReceipt:
+        draft = cls.model_construct(
+            receipt_id="sha256:" + "0" * 64,
+            run_id=run_id,
+            side=side,
+            policy_digest=policy_digest,
+            controls_digest=controls_digest,
+            evaluated_commit=evaluated_commit,
+            evaluated_tree=evaluated_tree,
+            task_ids=task_ids,
+            outcome_receipts=outcome_receipts,
+            blockers=blockers,
+        )
+        return cls(
+            receipt_id=draft.recomputed_id(),
+            run_id=run_id,
+            side=side,
+            policy_digest=policy_digest,
+            controls_digest=controls_digest,
+            evaluated_commit=evaluated_commit,
+            evaluated_tree=evaluated_tree,
+            task_ids=task_ids,
+            outcome_receipts=outcome_receipts,
+            blockers=blockers,
+        )
+
+    def recomputed_id(self) -> str:
+        canonical = self.model_dump_json(exclude={"receipt_id"})
+        return f"sha256:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()}"
+
+    @model_validator(mode="after")
+    def validate_identity_and_partition(self) -> EvaluatedRunReceipt:
+        if self.receipt_id != self.recomputed_id():
+            raise ValueError("receipt_id does not match canonical receipt content")
+        task_ids = self.task_ids
+        _validate_unique_ids(task_ids, "task_ids")
+        outcome_ids = tuple(item.task_id for item in self.outcome_receipts)
+        blocker_ids = tuple(item.task_id for item in self.blockers)
+        all_result_ids = outcome_ids + blocker_ids
+        _validate_partition(all_result_ids, task_ids)
+        _validate_result_order(outcome_ids, task_ids)
+        _validate_result_order(blocker_ids, task_ids)
+        return self
+
+
+def _validate_verdict_score(verdict: VerifierVerdict, score: float | None) -> None:
+    expected = (
+        1.0
+        if verdict is VerifierVerdict.PASS
+        else 0.0
+        if verdict is VerifierVerdict.FAIL
+        else None
+    )
+    if score != expected:
+        raise ValueError("normalized_score does not match verdict")
+
+
+def _validate_partition(
+    result_ids: tuple[str, ...],
+    task_ids: tuple[str, ...],
+) -> None:
+    _validate_unique_ids(result_ids, "outcomes and blockers")
+    if set(result_ids) != set(task_ids):
+        raise ValueError("outcomes and blockers must partition task_ids")
+
+
+def _validate_unique_ids(values: tuple[str, ...], field: str) -> None:
+    if len(set(values)) != len(values):
+        raise ValueError(f"{field} must be unique")
+
+
+def _validate_result_order(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> None:
+    if not _follows_task_order(result_ids, task_ids):
+        raise ValueError("outcomes and blockers must follow task_ids order")
+
+
+def _validate_run_metric(
+    value: float | None,
+    minimum: float,
+    maximum: float,
+    field: str,
+) -> None:
+    if value is not None and (not math.isfinite(value) or not minimum <= value <= maximum):
+        raise ValueError(f"{field} is outside its finite bounds")
+
+
+def _follows_task_order(
+    result_ids: tuple[str, ...],
+    task_ids: tuple[str, ...],
+) -> bool:
+    positions = tuple(task_ids.index(task_id) for task_id in result_ids)
+    return positions == tuple(sorted(positions))
+
+
 @dataclass(frozen=True, slots=True)
 class EvidenceReference:
     value: str