[codex] PR1: record evidence-backed harness hypotheses

plugins/openflywheel/.codex-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "openflywheel",
-  "version": "0.7.1",
-  "description": "Initialize ITSM-bench workspaces, mine exact failure patterns, and record evidence-bound curations.",
+  "version": "0.8.0",
+  "description": "Prepare ITSM-bench workspaces, record hypotheses, and execute isolated candidates.",
   "author": {
     "name": "OpenFlyWheel"
   },
@@ -10,14 +10,14 @@
   "skills": "./skills/",
   "interface": {
     "displayName": "OpenFlyWheel",
-    "shortDescription": "Mine and curate ITSM failure patterns",
-    "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, and persist evidence-bound cross-task curations without copying trace payloads.",
+    "shortDescription": "Execute isolated harness candidates",
+    "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, persist one evidence-backed hypothesis, execute one isolated candidate, and stop before admission without copying trace payloads.",
     "developerName": "OpenFlyWheel",
     "category": "Productivity",
     "capabilities": ["Read", "Write"],
     "defaultPrompt": [
       "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.",
-      "When PROGRAM.md is ready, read it and continue the optimization loop until its declared goal or stop condition is met."
+      "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, execute one isolated candidate, and stop before admission."
     ]
   },
   "mcpServers": "./.mcp.json"

plugins/openflywheel/.mcp.json

@@ -4,7 +4,7 @@
       "command": "uvx",
       "args": [
         "--from",
-        "git+https://github.com/divo12/OpenFlyWheel.git@b4c785d9472ca1d6c245c41fa4d9985f0cef97fc",
+        "git+https://github.com/divo12/OpenFlyWheel.git@9041db3c08a89df0fe9f8f2476a303b46dd2812a",
         "--with",
         "mcp>=1.13,<2",
         "openflywheel-mcp"

plugins/openflywheel/program_templates/base.md

@@ -4,16 +4,19 @@ This file is generated by `prepare_workspace`. Do not edit it directly.
 
 ## Mission
 
-Improve the connected agent harness until the goal in `experiment_config.yaml` is met,
-while respecting its quality, cost, latency, budget, and stopping constraints.
+Record one evidence-backed hypothesis, execute its isolated candidate under the canonical
+experiment policy, then stop before admission.
 
 The baseline has already been recorded. Begin at step 2; do not rerun the unchanged
-baseline.
+baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an
+existing terminal Harbor job was adopted).
 
 ## Authority
 
-- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal,
-  benchmark, verifier, budget, and stopping conditions.
+- The canonical experiment policy stored in the Git common control directory defines the
+  prepared branch and commits, exact editable surface, frozen controls, goal, benchmark,
+  verifier, budget, and stopping conditions. `experiment_config.yaml` is a human-readable copy,
+  not an internal authority interface.
 - The external verifier is authoritative for task outcome. The agent's completion claim
   is not proof of success.
 - Langfuse is the source of truth for trajectories, usage, cost, and latency.
@@ -24,9 +27,9 @@ baseline.
 
 ## Editable and frozen surfaces
 
-Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark,
-held-out tasks, verifier, model, reasoning budget, observability identity, or this program
-to improve a score.
+Target only exact paths allowed by the canonical experiment policy. Edit them only in the
+candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks,
+verifier, model, reasoning budget, observability identity, or this program.
 
 Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control
 flow changes unless the evidence requires the combination.
@@ -46,37 +49,22 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn
 expected improvement, and possible regressions. Stop if the evidence cannot distinguish
 between materially different changes.
 
-### 4. Improve the harness
+Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported
+pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call
+`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution.
 
-Make the smallest change within the declared editable surface. Preserve frozen controls and
-unrelated user changes.
+### 4. Execute one candidate
 
-### 5. Gate the change
+Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling
+candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated
+worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the
+returned candidate worktree, then call `execute_candidate` again with the identical request.
 
-Run only the prepared experiment command and gates declared by the workspace. Compare
-task-level verifier outcomes and report quality, cost, and latency separately. Missing or
-errored trials remain visible and cannot disappear from the denominator.
+Poll identical requests while the candidate is running. Retain its candidate ID, Git commit,
+trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally,
+change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree.
 
-### 6. Keep or revert
+## Package boundary
 
-Keep the change only when the configured gate admits it. Otherwise revert only the current
-iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken
-the gate to admit a candidate.
-
-Commit each admitted improvement on the prepared `ofw/<experiment-id>` branch before the
-next iteration. Keep one hypothesis per commit and include `OFW-Experiment` and `OFW-Run`
-trailers. Do not commit failed candidates, generated run artifacts, credentials, or changes
-outside the editable surface. Do not push or open a pull request without explicit user
-authorization.
-
-### 7. Repeat
-
-Return to step 2 with the newly recorded run. Stop when the configured goal is met, the
-budget or iteration limit is exhausted, the no-improvement condition is reached, or required
-authoritative evidence is unavailable.
-
-## Final report
-
-Report the accepted harness revision, verifier-backed quality, cost, latency, remaining
-unverified trials, iteration count, and exact stopping reason. Do not claim improvement from
-an agent-authored summary alone.
+Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission:
+do not gate, accept, merge, publish, push, or install the candidate.

plugins/openflywheel/skills/hypothesis-former/SKILL.md

@@ -0,0 +1,25 @@
+---
+name: hypothesis-former
+description: Record one OpenFlywheel harness hypothesis from one curated failure group and its exact supported pattern and diagnosis receipts. Use after failure curation and before any candidate edit; do not use with incomplete evidence or to modify harness files.
+---
+
+# Hypothesis Former
+
+Select one group from the retained failure-curation receipt. Use every diagnosis artifact ID in
+that group, partitioned under its exact mined pattern IDs; do not add or omit IDs. Keep the global
+diagnosis set at fifty IDs or fewer. Read the compact diagnoses, then propose one bounded
+statement, rationale, expected effect, regression-risk list, component taxonomy, and one or more
+exact target paths within the group's target component, explicit predicted task IDs, and at-risk
+task IDs. Keep those prediction sets disjoint so the next iteration can falsify the decision.
+
+Call `record_hypothesis` once with the prepared worktree, experiment ID, current initialization
+commit, curation ID, selected curation group ID, exact pattern-to-diagnosis assignments, explicit
+predicted and at-risk task IDs, and proposed target. The tool reloads the curation, requires the complete selected group, recomputes
+the patterns, and rejects missing, extra, misassigned, or inconclusive evidence. It also proves
+each target is exactly present in the authoritative editable allowlist; `component_kind` must
+match the curated component and never grants path access.
+
+Retain the returned hypothesis ID and artifact path. Stop before editing any harness file. On an
+MCP timeout, retry the identical request once because publication is idempotent; if the retry
+also times out, stop with unknown operation status. Do not query Langfuse, rewrite diagnoses,
+broaden target paths, infer support, or begin candidate work while following this skill.

plugins/openflywheel/skills/workspace-init/SKILL.md

@@ -25,7 +25,10 @@ when the evidence supports one. Collect, in order:
    directories.
 2. Optimization goal, primary metric, target, and stopping condition. Keep quality, cost,
    and latency constraints separate rather than hiding them in one average.
-3. ITSM-bench root, Harbor executable, Harbor configuration, and expected task count.
+3. ITSM-bench root, Harbor executable, Harbor configuration, and expected task count. If an
+   exact terminal Harbor job already exists at `benchmark_root/jobs/<experiment-id>`, ask whether
+   to adopt it; set `reuse_existing_baseline=true` only after explicit confirmation so policy is
+   published without launching a duplicate baseline.
 4. Experiment ID and maximum baseline duration.
 
 Read and report the frozen model from the Harbor configuration. `prepare_workspace` fixes

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "openflywheel"
-version = "0.7.1"
+version = "0.8.0"
 description = "A governed self-improving agent harness"
 requires-python = ">=3.11"
 dependencies = [

src/ofw/__init__.py

@@ -13,6 +13,9 @@
 from ofw.contracts import ComponentKind, Sha256Digest
 from ofw.evaluation import (
     DeferredFailure,
+    EvaluatedRunBlocker,
+    EvaluatedRunReceipt,
+    EvaluatedTaskReceipt,
     EvidenceReference,
     FailureCuration,
     FailureCurationErrorCode,
@@ -39,11 +42,33 @@
     OutcomeScoreSubmission,
     OutcomeStoreObservation,
     OutcomeStoreStatus,
+    RunSide,
     TaskId,
     VerifierId,
     VerifierResult,
     VerifierVerdict,
 )
+from ofw.evolution import (
+    CandidateBlockerCode,
+    CandidateErrorCode,
+    CandidateExecutionInput,
+    CandidateExecutionObservation,
+    CandidateFailure,
+    CandidateId,
+    CandidatePhase,
+    CandidateStatus,
+    FailurePatternReference,
+    FailurePatternReferenceInput,
+    HarnessChangeTarget,
+    HarnessChangeTargetInput,
+    HarnessHypothesis,
+    HypothesisErrorCode,
+    HypothesisFailure,
+    HypothesisId,
+    HypothesisObservation,
+    HypothesisStatus,
+    RecordHypothesisInput,
+)
 from ofw.observability.langfuse import (
     CollectionError,
     CollectionErrorCode,
@@ -52,6 +77,9 @@
 )
 from ofw.observability.langfuse.domain import TraceId
 from ofw.preparation import (
+    ExperimentPolicyErrorCode,
+    ExperimentPolicyFailure,
+    ExperimentPolicySnapshot,
     PreparationErrorCode,
     PreparationPhase,
     PreparationStatus,
@@ -60,11 +88,25 @@
 )
 
 __all__ = [
+    "CandidateBlockerCode",
+    "CandidateErrorCode",
+    "CandidateExecutionInput",
+    "CandidateExecutionObservation",
+    "CandidateFailure",
+    "CandidateId",
+    "CandidatePhase",
+    "CandidateStatus",
     "CollectionError",
     "CollectionErrorCode",
     "ComponentKind",
     "DeferredFailure",
     "EvidenceReference",
+    "EvaluatedRunBlocker",
+    "EvaluatedRunReceipt",
+    "EvaluatedTaskReceipt",
+    "ExperimentPolicyErrorCode",
+    "ExperimentPolicyFailure",
+    "ExperimentPolicySnapshot",
     "FailureCuration",
     "FailureCurationErrorCode",
     "FailureCurationFailure",
@@ -80,8 +122,18 @@
     "FailurePatternMiningStatus",
     "FailurePatternOrdering",
     "FailurePatternSummary",
+    "FailurePatternReference",
+    "FailurePatternReferenceInput",
     "FailureSource",
     "FailureType",
+    "HarnessChangeTarget",
+    "HarnessChangeTargetInput",
+    "HarnessHypothesis",
+    "HypothesisErrorCode",
+    "HypothesisFailure",
+    "HypothesisId",
+    "HypothesisObservation",
+    "HypothesisStatus",
     "Langfuse",
     "LangfuseOtelSpanAttributes",
     "LangfuseOutcomeStore",
@@ -94,10 +146,12 @@
     "OutcomeScoreSubmission",
     "OutcomeStoreObservation",
     "OutcomeStoreStatus",
+    "RunSide",
     "PreparationErrorCode",
     "PreparationPhase",
     "PreparationStatus",
     "PrepareWorkspaceInput",
+    "RecordHypothesisInput",
     "Sha256Digest",
     "TaskId",
     "TraceId",

src/ofw/evaluation/__init__.py

@@ -32,10 +32,14 @@
     OutcomeStoreStatus,
 )
 from ofw.evaluation.outcome import (
+    EvaluatedRunBlocker,
+    EvaluatedRunReceipt,
+    EvaluatedTaskReceipt,
     EvidenceReference,
     OutcomeErrorCode,
     OutcomeEvaluation,
     OutcomeEvaluationError,
+    RunSide,
     TaskId,
     VerifierId,
     VerifierResult,
@@ -45,6 +49,9 @@
 __all__ = [
     "DeferredFailure",
     "EvidenceReference",
+    "EvaluatedRunBlocker",
+    "EvaluatedRunReceipt",
+    "EvaluatedTaskReceipt",
     "FailureCuration",
     "FailureCurationErrorCode",
     "FailureCurationFailure",
@@ -67,6 +74,7 @@
     "OutcomeErrorCode",
     "OutcomeEvaluation",
     "OutcomeEvaluationError",
+    "RunSide",
     "OutcomeScoreSubmission",
     "OutcomeStoreObservation",
     "OutcomeStoreStatus",